mikefarah/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2026-10-11 21:39:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master', remote-tracking branch 'origin' into fix/2827-base64url-tests

Browse Source
This commit is contained in:
Mike Farah
2026-09-15 15:54:23 +10:00
parent 41f5cfe1d2 5efa5ffcd8 0442602b32
commit 55ee81bf21
37 changed files with 494 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/docker-githubaction.yml
+1 -1
View File
@@ -24,7 +24,7 @@ jobs:
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
with:
version: latest
.github/workflows/docker-release.yml
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
with:
version: latest
.github/workflows/release.yml
+1 -1
View File
@@ -48,7 +48,7 @@ jobs:
- name: Cross compile
run: |
sudo apt-get install rhash -y
go install github.com/goreleaser/goreleaser/[email protected]6.0
go install github.com/goreleaser/goreleaser/[email protected]7.1
./scripts/xcompile.sh
- name: Sign checksums
.github/workflows/scorecard.yml
+2 -2
View File
@@ -39,7 +39,7 @@ jobs:
persist-credentials: false
- name: "Run analysis"
uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
with:
results_file: results.sarif
results_format: sarif
@@ -73,6 +73,6 @@ jobs:
# Upload the results to GitHub's code scanning dashboard (optional).
# Commenting out will disable upload of results to your repo's Code Scanning dashboard
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5
with:
sarif_file: results.sarif
Dockerfile
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.26.5@sha256:079e59808d2d252516e27e3f3a9c003740dee7f75e55aa71528766d52bcfc16a AS builder
FROM golang:1.26.6@sha256:0d1d3a794be25f809dd2cb3160d8c73276c4056a9f8242a138e908ddeee7b6b6 AS builder
WORKDIR /go/src/mikefarah/yq
Dockerfile.dev
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.26.5@sha256:079e59808d2d252516e27e3f3a9c003740dee7f75e55aa71528766d52bcfc16a
FROM golang:1.26.6@sha256:0d1d3a794be25f809dd2cb3160d8c73276c4056a9f8242a138e908ddeee7b6b6
COPY scripts/devtools.sh /opt/devtools.sh
cmd/utils.go
+5 -3
View File
@@ -7,6 +7,7 @@ import (
"os"
"strings"
"github.com/mattn/go-isatty"
"github.com/mikefarah/yq/v4/pkg/yqlib"
"github.com/spf13/cobra"
)
@@ -45,9 +46,10 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
}
func setupColors() {
fileInfo, _ := os.Stdout.Stat()
if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) {
fd := os.Stdout.Fd()
// os.Stdout can be a regular file, pipe, or other character device such
// as /dev/null - only a real terminal should get colourised output.
if forceColor || (!forceNoColor && (isatty.IsTerminal(fd) || isatty.IsCygwinTerminal(fd))) {
colorsEnabled = true
}
}
cmd/utils_test.go
+100
View File
@@ -3,7 +3,10 @@ package cmd
import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"syscall"
"testing"
"github.com/mikefarah/yq/v4/pkg/yqlib"
@@ -964,6 +967,103 @@ func TestSetupColors(t *testing.T) {
}
}
// terminalCase is a file/device to test setupColors against, paired with
// whether it should be treated as an interactive terminal - so what does
// and doesn't get colourised output is visible in a single place.
type terminalCase struct {
open func(t *testing.T) *os.File
isTerm bool
}
func terminalTestCases() map[string]terminalCase {
return map[string]terminalCase{
"regular file": {isTerm: false, open: func(t *testing.T) *os.File {
f, err := os.CreateTemp(t.TempDir(), "isterminal")
if err != nil {
t.Fatalf("failed to create temp file: %v", err)
}
return f
}},
"/dev/null": {isTerm: false, open: func(t *testing.T) *os.File {
f, err := os.OpenFile(os.DevNull, os.O_RDWR, 0644)
if err != nil {
t.Fatalf("failed to open %s: %v", os.DevNull, err)
}
return f
}},
"unnamed pipe, read end": {isTerm: false, open: func(t *testing.T) *os.File {
r, w, err := os.Pipe()
if err != nil {
t.Fatalf("failed to create pipe: %v", err)
}
w.Close()
return r
}},
"unnamed pipe, write end (e.g. piped to a pager)": {isTerm: false, open: func(t *testing.T) *os.File {
r, w, err := os.Pipe()
if err != nil {
t.Fatalf("failed to create pipe: %v", err)
}
r.Close()
return w
}},
"named pipe (FIFO)": {isTerm: false, open: func(t *testing.T) *os.File {
if runtime.GOOS == "windows" {
t.Skip("named pipes not supported by this test on windows")
}
path := filepath.Join(t.TempDir(), "fifo")
if err := syscall.Mkfifo(path, 0600); err != nil {
t.Fatalf("failed to create named pipe: %v", err)
}
// O_RDWR avoids blocking on open: a FIFO opened only for reading
// (or only writing) blocks until a peer opens the other end.
f, err := os.OpenFile(path, os.O_RDWR, os.ModeNamedPipe)
if err != nil {
t.Fatalf("failed to open named pipe: %v", err)
}
return f
}},
"controlling terminal (/dev/tty)": {isTerm: true, open: func(t *testing.T) *os.File {
f, err := os.OpenFile("/dev/tty", os.O_RDWR, 0)
if err != nil {
t.Skipf("no controlling terminal available in this environment: %v", err)
}
return f
}},
}
}
func TestSetupColorsForFileKind(t *testing.T) {
for name, tc := range terminalTestCases() {
t.Run(name, func(t *testing.T) {
f := tc.open(t)
defer f.Close()
originalStdout := os.Stdout
originalForceColor := forceColor
originalForceNoColor := forceNoColor
originalColorsEnabled := colorsEnabled
defer func() {
os.Stdout = originalStdout
forceColor = originalForceColor
forceNoColor = originalForceNoColor
colorsEnabled = originalColorsEnabled
}()
os.Stdout = f
forceColor = false
forceNoColor = false
colorsEnabled = false
setupColors()
if colorsEnabled != tc.isTerm {
t.Errorf("setupColors() colorsEnabled = %v, want %v for %s", colorsEnabled, tc.isTerm, name)
}
})
}
}
func TestLoadSplitFileExpression(t *testing.T) {
// Create a temporary file with expression content
tempFile, err := os.CreateTemp("", "split")
cmd/version.go
+1 -1
View File
@@ -11,7 +11,7 @@ var (
GitDescribe string
// Version is main version number that is being run at the moment.
Version = "v4.53.3"
Version = "v4.53.6"
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
go.mod
+7 -7
View File
@@ -3,7 +3,7 @@ module github.com/mikefarah/yq/v4
require (
github.com/a8m/envsubst v1.4.3
github.com/alecthomas/participle/v2 v2.1.4
github.com/alecthomas/repr v0.5.2
github.com/alecthomas/repr v0.5.4
github.com/dimchansky/utfbom v1.1.1
github.com/elliotchance/orderedmap v1.8.0
github.com/fatih/color v1.19.0
@@ -12,7 +12,8 @@ require (
github.com/goccy/go-yaml v1.19.2
github.com/hashicorp/hcl/v2 v2.24.0
github.com/jinzhu/copier v0.4.0
github.com/magiconair/properties v1.8.10
github.com/magiconair/properties v1.18.11
github.com/mattn/go-isatty v0.0.20
github.com/pelletier/go-toml/v2 v2.4.3
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
github.com/spf13/cobra v1.10.2
@@ -20,9 +21,9 @@ require (
github.com/yuin/gopher-lua v1.1.2
github.com/zclconf/go-cty v1.19.0
go.yaml.in/yaml/v4 v4.0.0-rc.6
golang.org/x/mod v0.38.0
golang.org/x/net v0.57.0
golang.org/x/text v0.40.0
golang.org/x/mod v0.40.0
golang.org/x/net v0.58.0
golang.org/x/text v0.41.0
)
require (
@@ -32,11 +33,10 @@ require (
github.com/google/go-cmp v0.6.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/tools v0.47.0 // indirect
golang.org/x/tools v0.49.0 // indirect
)
go 1.25.0
go.sum
+12 -12
View File
@@ -6,8 +6,8 @@ github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8v
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U=
github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI=
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/alecthomas/repr v0.5.4 h1:OVP7JEcuzU9CCDsT6STCr3rg17oQfWILtPWd2EG0uN4=
github.com/alecthomas/repr v0.5.4/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
github.com/apparentlymart/go-textseg/v17 v17.0.1 h1:bpMXRgQ5cEoRNuQke1a80/Nl6w3G5eoIbWo9f3gXkAs=
@@ -40,8 +40,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/magiconair/properties v1.18.11 h1:j5ozYZl0zCjG7ahMDH0GWIobOvvUzT0BdAguG0ViKy0=
github.com/magiconair/properties v1.18.11/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -72,19 +72,19 @@ github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmB
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
pkg/yqlib/all_at_once_evaluator.go
+2 -1
View File
@@ -49,12 +49,13 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
var allDocuments = list.New()
for _, filename := range filenames {
reader, err := readStream(filename)
reader, cleanup, err := readStream(filename)
if err != nil {
return err
}
fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder)
cleanup()
if err != nil {
return err
}
pkg/yqlib/decoder_hcl.go
+3 -5
View File
@@ -300,8 +300,7 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode
it := v.ElementIterator()
for it.Next() {
key, val := it.Element()
keyStr := key.AsString()
keyNode := createStringScalarNode(keyStr)
keyNode := convertCtyValueToNode(key)
valNode := convertCtyValueToNode(val)
m.AddKeyValueChild(keyNode, valNode)
}
@@ -338,8 +337,7 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode
}
continue
}
keyStr := keyVal.AsString()
keyNode := createStringScalarNode(keyStr)
keyNode := convertCtyValueToNode(keyVal)
valNode := convertHclExprToNode(item.ValueExpr, src)
m.AddKeyValueChild(keyNode, valNode)
}
@@ -460,7 +458,7 @@ func convertCtyValueToNode(v cty.Value) *CandidateNode {
it := v.ElementIterator()
for it.Next() {
key, val := it.Element()
keyNode := createStringScalarNode(key.AsString())
keyNode := convertCtyValueToNode(key)
valNode := convertCtyValueToNode(val)
m.AddKeyValueChild(keyNode, valNode)
}
pkg/yqlib/doc/operators/system-operators.md
+2 -2
View File
@@ -51,7 +51,7 @@ country: Australia
```
then
```bash
yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml
yq --security-enable-system-operator '.country = system("/bin/echo"; "test")' sample.yml
```
will output
```yaml
@@ -67,7 +67,7 @@ a: hello
```
then
```bash
yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml
yq --security-enable-system-operator '.a = system("/bin/echo")' sample.yml
```
will output
```yaml
pkg/yqlib/encoder_yaml.go
+21 -6
View File
@@ -2,6 +2,7 @@ package yqlib
import (
"bytes"
"fmt"
"io"
"strings"
@@ -49,23 +50,37 @@ func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
destination = tempBuffer
}
var encoder = yaml.NewEncoder(destination)
indent := ye.prefs.Indent
if indent < 2 {
indent = 2
} else if indent > 9 {
indent = 9
}
encoder.SetIndent(ye.prefs.Indent)
if ye.prefs.CompactSequenceIndent {
encoder.CompactSeqIndent()
dumper, err := yaml.NewDumper(destination,
yaml.WithV3Defaults(),
yaml.WithIndent(indent),
yaml.WithCompactSeqIndent(ye.prefs.CompactSequenceIndent),
yaml.WithLineWidth(-1),
)
if err != nil {
return fmt.Errorf("configure YAML encoding: %w", err)
}
target, err := node.MarshalYAML()
if err != nil {
_ = dumper.Close()
return err
}
trailingContent := target.FootComment
target.FootComment = ""
if err := encoder.Encode(target); err != nil {
err = dumper.Dump(target)
if closeErr := dumper.Close(); err == nil {
err = closeErr
}
if err != nil {
return err
}
pkg/yqlib/front_matter_test.go
+2 -1
View File
@@ -136,10 +136,11 @@ Some content
test.AssertResult(t, originalFilename, resolved)
// Read documents using the temp file, verify they get the original filename
reader, err := readStream(tempFilename)
reader, cleanup, err := readStream(tempFilename)
if err != nil {
panic(err)
}
defer cleanup()
decoder := NewYamlDecoder(ConfiguredYamlPreferences)
docs, err := readDocuments(reader, tempFilename, 0, decoder)
if err != nil {
pkg/yqlib/hcl_test.go
+42
View File
@@ -176,6 +176,41 @@ var hclFormatScenarios = []formatScenario{
expected: "obj: {a: 1, b: \"two\"}\n",
scenarioType: "decode",
},
{
description: "object with integer keys",
skipDoc: true,
input: `obj = { 1 = "one", 2 = "two" }`,
expected: "obj: {1: \"one\", 2: \"two\"}\n",
scenarioType: "decode",
},
{
description: "object with boolean keys",
skipDoc: true,
input: `obj = { (true) = "yes", (false) = "no" }`,
expected: "obj: {true: \"yes\", false: \"no\"}\n",
scenarioType: "decode",
},
{
description: "object with float keys",
skipDoc: true,
input: `obj = { (3.14) = "pi" }`,
expected: "obj: {3.14: \"pi\"}\n",
scenarioType: "decode",
},
{
description: "object with mixed scalar keys",
skipDoc: true,
input: `obj = { a = 1, 1 = "one", (true) = "yes" }`,
expected: "obj: {a: 1, 1: \"one\", true: \"yes\"}\n",
scenarioType: "decode",
},
{
description: "nested object with integer keys",
skipDoc: true,
input: `config = { levels = { 1 = "debug", 2 = "info" } }`,
expected: "config: {levels: {1: \"debug\", 2: \"info\"}}\n",
scenarioType: "decode",
},
{
description: "nested block",
skipDoc: true,
@@ -472,6 +507,13 @@ var hclFormatScenarios = []formatScenario{
expected: "service {\n optional_field = null\n}\n",
scenarioType: "roundtrip",
},
{
description: "object with integer key and empty value",
skipDoc: true,
input: `intdict = { 1 = {} }`,
expected: "intdict: {1: {}}\n",
scenarioType: "decode",
},
}
func testHclScenario(t *testing.T, s formatScenario) {
pkg/yqlib/lib.go
+29
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"log/slog"
"math"
"math/big"
"strconv"
"strings"
"sync"
@@ -204,6 +205,34 @@ func parseInt(numberString string) (int, error) {
return int(parsed), err
}
// checkedInt64Arithmetic performs '+', '-' or '*' on two int64 values and
// returns the exact result only when it still fits in an int64. yq refuses to
// silently wrap on int64 overflow (see issue #2820): a result outside
// [math.MinInt64, math.MaxInt64] is reported as an error rather than returned
// as a wrapped (and therefore wrong) value. big.Int is used so the exact
// product/sum/difference is computed before the range check, which avoids the
// sign-handling pitfalls of doing the overflow detection directly on the
// wrapped int64 result.
func checkedInt64Arithmetic(op byte, lhsNum, rhsNum int64) (int64, error) {
lhs := new(big.Int).SetInt64(lhsNum)
rhs := new(big.Int).SetInt64(rhsNum)
result := new(big.Int)
switch op {
case '+':
result.Add(lhs, rhs)
case '-':
result.Sub(lhs, rhs)
case '*':
result.Mul(lhs, rhs)
default:
return 0, fmt.Errorf("unknown arithmetic operator %q", op)
}
if !result.IsInt64() {
return 0, fmt.Errorf("%v %c %v overflows int64", lhsNum, op, rhsNum)
}
return result.Int64(), nil
}
func processEscapeCharacters(original string) string {
if original == "" {
return original
pkg/yqlib/lib_test.go
+40
View File
@@ -2,6 +2,7 @@ package yqlib
import (
"fmt"
"math"
"strings"
"sync"
"testing"
@@ -173,6 +174,45 @@ func TestParseInt64(t *testing.T) {
}
}
func TestCheckedInt64Arithmetic(t *testing.T) {
type checkedInt64ArithmeticScenario struct {
description string
op byte
lhs int64
rhs int64
expectedResult int64
expectedError string
}
scenarios := []checkedInt64ArithmeticScenario{
{description: "add, no overflow", op: '+', lhs: 3, rhs: 4, expectedResult: 7},
{description: "add, overflows MaxInt64", op: '+', lhs: math.MaxInt64, rhs: 1, expectedError: "9223372036854775807 + 1 overflows int64"},
{description: "subtract, no overflow", op: '-', lhs: 10, rhs: 3, expectedResult: 7},
{description: "subtract, underflows MinInt64", op: '-', lhs: math.MinInt64, rhs: 1, expectedError: "-9223372036854775808 - 1 overflows int64"},
{description: "multiply, no overflow", op: '*', lhs: 6, rhs: 7, expectedResult: 42},
{description: "multiply, overflows MaxInt64", op: '*', lhs: math.MaxInt64, rhs: 2, expectedError: "9223372036854775807 * 2 overflows int64"},
{description: "multiply, MinInt64 by -1 overflows (no positive counterpart exists)", op: '*', lhs: math.MinInt64, rhs: -1, expectedError: "-9223372036854775808 * -1 overflows int64"},
{description: "multiply, MinInt64 by 1 does not overflow", op: '*', lhs: math.MinInt64, rhs: 1, expectedResult: math.MinInt64},
{description: "unknown operator", op: '/', lhs: 4, rhs: 2, expectedError: "unknown arithmetic operator"},
}
for _, tt := range scenarios {
actualResult, err := checkedInt64Arithmetic(tt.op, tt.lhs, tt.rhs)
if tt.expectedError != "" {
if err == nil {
t.Errorf("%v: expected error containing '%s' but got none", tt.description, tt.expectedError)
} else if !strings.Contains(err.Error(), tt.expectedError) {
t.Errorf("%v: expected error containing '%s', got '%s'", tt.description, tt.expectedError, err.Error())
}
continue
}
if err != nil {
t.Errorf("%v: unexpected error: %v", tt.description, err)
}
test.AssertResultComplexWithContext(t, tt.expectedResult, actualResult, tt.description)
}
}
func TestGetContentValueByKey(t *testing.T) {
// Create content with key-value pairs
key1 := createStringScalarNode("key1")
pkg/yqlib/operator_add.go
+4 -1
View File
@@ -122,7 +122,10 @@ func addScalars(context Context, target *CandidateNode, lhs *CandidateNode, rhs
if err != nil {
return err
}
sum := lhsNum + rhsNum
sum, err := checkedInt64Arithmetic('+', lhsNum, rhsNum)
if err != nil {
return err
}
target.Tag = lhs.Tag
target.Value = fmt.Sprintf(format, sum)
} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
pkg/yqlib/operator_add_test.go
+9
View File
@@ -539,6 +539,15 @@ var addOperatorScenarios = []expressionScenario{
"D0, P[], (!!map)::? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2\n",
},
},
{
// int64 + int64 that overflows int64. See issue #2820: previously
// this silently wrapped (MaxInt64 + 1 -> MinInt64).
description: "Add integers that overflow int64",
skipDoc: true,
document: `a: 9223372036854775807`,
expression: `.a + 1`,
expectedError: "9223372036854775807 + 1 overflows int64",
},
}
func TestAddOperatorScenarios(t *testing.T) {
pkg/yqlib/operator_load_test.go
+1 -1
View File
@@ -77,7 +77,7 @@ var loadScenarios = []expressionScenario{
document: `{something: {file: "thing.yml"}, over: {here: [{file: "thing.yml"}]}}`,
expression: `(.. | select(has("file"))) |= load("../../examples/" + .file)`,
expected: []string{
"D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included,\n b: cool.}]}}\n",
"D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included, b: cool.}]}}\n",
},
},
{
pkg/yqlib/operator_multiply.go
+5 -1
View File
@@ -133,7 +133,11 @@ func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, e
if err != nil {
return nil, err
}
target.Value = fmt.Sprintf(format, lhsNum*rhsNum)
product, err := checkedInt64Arithmetic('*', lhsNum, rhsNum)
if err != nil {
return nil, err
}
target.Value = fmt.Sprintf(format, product)
return target, nil
}
pkg/yqlib/operator_multiply_test.go
+19 -2
View File
@@ -83,7 +83,7 @@ var nodeWithFooter = `a: apple
var document = `a: &cat {name: cat}
b: {name: dog}
c:
c:
<<: *cat
`
@@ -514,7 +514,7 @@ var multiplyOperatorScenarios = []expressionScenario{
environmentVariables: map[string]string{"originalPath": ".myArray", "otherPath": ".newArray", "idPath": ".a"},
expression: mergeExpression,
expected: []string{
"D0, P[], (!!map)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC},\n {a: dingo, c: dingoC}], something: else}\n",
"D0, P[], (!!map)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\n",
},
},
{
@@ -716,6 +716,23 @@ var multiplyOperatorScenarios = []expressionScenario{
expression: fmt.Sprintf(`"ab" * %d`, 1<<(bits.UintSize-2)),
expectedError: fmt.Sprintf("result of repeating string (2 bytes) by %d would exceed 10485760 bytes", 1<<(bits.UintSize-2)),
},
{
// int64 * int64 that overflows int64. See issue #2820: previously
// this silently wrapped to a wrong (negative) value.
description: "Multiply integers that overflow int64",
skipDoc: true,
document: `a: 3000000000`,
expression: `.a * 4000000000`,
expectedError: "3000000000 * 4000000000 overflows int64",
},
{
// MaxInt64 * 2 wraps to -2 without the overflow guard.
description: "Multiply max int by 2 overflows int64",
skipDoc: true,
document: `a: 9223372036854775807`,
expression: `.a * 2`,
expectedError: "9223372036854775807 * 2 overflows int64",
},
}
func TestMultiplyOperatorScenarios(t *testing.T) {
pkg/yqlib/operator_select_test.go
+1 -1
View File
@@ -98,7 +98,7 @@ var selectOperatorScenarios = []expressionScenario{
document: `[{animal: cat, legs: {cool: true}}, {animal: fish}]`,
expression: `(.[] | select(.legs.cool == true).canWalk) = true | (.[] | .alive.things) = "yes"`,
expected: []string{
"D0, P[], (!!seq)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish,\n alive: {things: yes}}]\n",
"D0, P[], (!!seq)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish, alive: {things: yes}}]\n",
},
},
{
pkg/yqlib/operator_sort_keys_test.go
+1 -1
View File
@@ -37,7 +37,7 @@ var sortKeysOperatorScenarios = []expressionScenario{
document: `{bParent: {c: dog, array: [3,1,2]}, aParent: {z: donkey, x: [{c: yum, b: delish}, {b: ew, a: apple}]}}`,
expression: `sort_keys(..)`,
expected: []string{
"D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [\n 3, 1, 2], c: dog}}\n",
"D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}}\n",
},
},
}
pkg/yqlib/operator_subtract.go
+4 -1
View File
@@ -104,7 +104,10 @@ func subtractScalars(context Context, target *CandidateNode, lhs *CandidateNode,
if err != nil {
return err
}
result := lhsNum - rhsNum
result, err := checkedInt64Arithmetic('-', lhsNum, rhsNum)
if err != nil {
return err
}
target.Tag = lhs.Tag
target.Value = fmt.Sprintf(format, result)
} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
pkg/yqlib/operator_subtract_test.go
+9
View File
@@ -157,6 +157,15 @@ var subtractOperatorScenarios = []expressionScenario{
"D0, P[], (!!seq)::[!goat {a: b}]\n",
},
},
{
// int64 - int64 that overflows int64. See issue #2820: previously
// this silently wrapped (MinInt64 - 1 -> MaxInt64).
description: "Subtract integers that overflow int64",
skipDoc: true,
document: `a: -9223372036854775808`,
expression: `.a - 1`,
expectedError: "-9223372036854775808 - 1 overflows int64",
},
}
func TestSubtractOperatorScenarios(t *testing.T) {
pkg/yqlib/stream_evaluator.go
+2 -7
View File
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"io"
"os"
)
// A yaml expression evaluator that runs the expression multiple times for each given yaml document.
@@ -50,21 +49,17 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
}
for _, filename := range filenames {
reader, err := readStream(filename)
reader, cleanup, err := readStream(filename)
if err != nil {
return err
}
processedDocs, err := s.Evaluate(filename, reader, node, printer, decoder)
cleanup()
if err != nil {
return err
}
totalProcessDocs = totalProcessDocs + processedDocs
switch reader := reader.(type) {
case *os.File:
safelyCloseFile(reader)
}
}
if totalProcessDocs == 0 {
pkg/yqlib/toml_test.go
+1 -1
View File
@@ -369,7 +369,7 @@ var tomlScenarios = []formatScenario{
skipDoc: true,
description: "blank",
input: `A = "hello`,
expectedError: `bad file 'sample.yml': basic string not terminated by "`,
expectedError: `bad file 'sample.yml': unterminated basic string`,
scenarioType: "decode-error",
},
{
pkg/yqlib/utils.go
+13 -14
View File
@@ -32,25 +32,24 @@ func resolveFilename(filename string) string {
return filename
}
func readStream(filename string) (io.Reader, error) {
var reader *bufio.Reader
// readStream returns a reader for the given file, along with a cleanup function
// that must be called once the reader is no longer needed. The cleanup is a no-op
// for stdin.
func readStream(filename string) (io.Reader, func(), error) {
if filename == "-" {
reader = bufio.NewReader(os.Stdin)
} else {
// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
// and ensuring that it's not possible to give a path to a file outside that directory.
file, err := os.Open(filename) // #nosec
if err != nil {
return nil, err
}
reader = bufio.NewReader(file)
return bufio.NewReader(os.Stdin), func() {}, nil
}
return reader, nil
// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
// and ensuring that it's not possible to give a path to a file outside that directory.
file, err := os.Open(filename) // #nosec
if err != nil {
return nil, nil, err
}
return bufio.NewReader(file), func() { safelyCloseFile(file) }, nil
}
func writeString(writer io.Writer, txt string) error {
_, errorWriting := writer.Write([]byte(txt))
_, errorWriting := io.WriteString(writer, txt)
return errorWriting
}
pkg/yqlib/utils_test.go
+119
View File
@@ -0,0 +1,119 @@
package yqlib
import (
"bufio"
"bytes"
"io"
"os"
"path/filepath"
"runtime/debug"
"strconv"
"testing"
"github.com/mikefarah/yq/v4/test"
)
// countOpenFileDescriptors returns the number of file descriptors this process
// currently holds open, or -1 if the platform does not expose them.
func countOpenFileDescriptors() int {
for _, dir := range []string{"/proc/self/fd", "/dev/fd"} {
// Readdirnames avoids stat-ing each entry, which races with descriptors
// (including this directory handle) being closed underneath us.
handle, err := os.Open(dir)
if err != nil {
continue
}
names, err := handle.Readdirnames(-1)
safelyCloseFile(handle)
if err != nil {
continue
}
// discount the directory handle itself
return len(names) - 1
}
return -1
}
func writeSampleFiles(t *testing.T, count int) []string {
t.Helper()
dir := t.TempDir()
filenames := make([]string, count)
for i := 0; i < count; i++ {
filename := filepath.Join(dir, "sample-"+strconv.Itoa(i)+".yml")
if err := os.WriteFile(filename, []byte("a: apple\n"), 0600); err != nil {
t.Fatalf("failed to write sample file: %v", err)
}
filenames[i] = filename
}
return filenames
}
func discardingPrinter() Printer {
return NewPrinter(NewYamlEncoder(ConfiguredYamlPreferences), NewSinglePrinterWriter(bufio.NewWriter(io.Discard)))
}
func assertNoLeakedFileDescriptors(t *testing.T, evaluate func(filenames []string) error) {
t.Helper()
InitExpressionParser()
// os.File finalisers close leaked descriptors on collection, which would
// let a genuine leak pass unnoticed.
defer debug.SetGCPercent(debug.SetGCPercent(-1))
before := countOpenFileDescriptors()
if before < 0 {
t.Skip("file descriptors are not observable on this platform")
}
if err := evaluate(writeSampleFiles(t, 50)); err != nil {
t.Fatalf("failed to evaluate files: %v", err)
}
after := countOpenFileDescriptors()
if after > before {
t.Errorf("expected no additional open file descriptors, had %d before and %d after", before, after)
}
}
func TestStreamEvaluatorClosesInputFiles(t *testing.T) {
assertNoLeakedFileDescriptors(t, func(filenames []string) error {
return NewStreamEvaluator().EvaluateFiles(".a", filenames, discardingPrinter(), NewYamlDecoder(ConfiguredYamlPreferences))
})
}
func TestAllAtOnceEvaluatorClosesInputFiles(t *testing.T) {
assertNoLeakedFileDescriptors(t, func(filenames []string) error {
return NewAllAtOnceEvaluator().EvaluateFiles(".a", filenames, discardingPrinter(), NewYamlDecoder(ConfiguredYamlPreferences))
})
}
// plainWriter only implements io.Writer, so io.WriteString must fall back to Write.
type plainWriter struct {
buf bytes.Buffer
}
func (w *plainWriter) Write(p []byte) (int, error) {
return w.buf.Write(p)
}
func TestWriteStringToStringWriter(t *testing.T) {
var buf bytes.Buffer
writer := bufio.NewWriter(&buf)
test.AssertResult(t, nil, writeString(writer, "hello world"))
test.AssertResult(t, nil, writer.Flush())
test.AssertResult(t, "hello world", buf.String())
}
func TestWriteStringToPlainWriter(t *testing.T) {
writer := &plainWriter{}
test.AssertResult(t, nil, writeString(writer, "hello world"))
test.AssertResult(t, "hello world", writer.buf.String())
}
func TestWriteStringDoesNotAllocate(t *testing.T) {
writer := bufio.NewWriter(io.Discard)
allocations := testing.AllocsPerRun(100, func() {
_ = writeString(writer, "hello world")
})
test.AssertResult(t, 0.0, allocations)
}
pkg/yqlib/yaml_test.go
+6
View File
@@ -106,6 +106,12 @@ var yamlFormatScenarios = []formatScenario{
input: "[1, 2]",
expected: "[1, 2]\n",
},
{
description: "long line",
skipDoc: true,
input: "field: Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt",
expected: "field: Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt\n",
},
}
var yamlParseScenarios = []expressionScenario{
release_notes.txt
+23
View File
@@ -1,3 +1,26 @@
4.53.6:
- Fixing release build issue
4.53.5:
- Fixed line wrapping bug (#2824, #2823) Thanks mxey
- Bumped dependencies
4.53.4:
- Close input files after processing each one (#2796) (#2808) Thanks @MsfPablo
- Fixed non string HCL string keys (#2795)
- Fixed heap allocation bug (#2809) Thanks @MsfPablo
- Fix deleting commented empty list YAML output (#2765) Thanks @dpersek
- Fix (has) return false for negative array indices (#2769) Thanks @maximilize
- Fix (sort): avoid int64 overflow comparing large integers (#2771) Thanks @maximilize
- Fix: preserve correct parent references in explode merge anchor reconstruction (#2730) Thanks @vomba
- Improve Guard ExpressionParser initialization with sync.Once (#2789) Thanks @arcaven
- Fix: reject negative indent instead of panicking (#2746) Thanks @StressTestor
- Fix: parse signed hex and octal integers (#2749) Thanks @StressTestor
- Fix: skip UTF-8 BOM when processing front matter (#2751) Thanks @StressTestor
- Fix !!merge tag regression for yq (#2705) Thanks @W-Floyd
- Default to yaml when a file's extension is not a recognised format (#2785) Thanks @devthedevil
- Bumped dependencies
4.53.3:
- Add `--ini-preserve-quotes` flag for INI round-trip quote preservation (#2728) Thanks @toller892!
- Fix: reset INI decoder state on init (#2719) Thanks @xieby1!
scripts/xcompile.sh
+1 -1
View File
@@ -2,7 +2,7 @@
set -eo pipefail
# You may need to go install github.com/goreleaser/goreleaser/[email protected]6.0 first
# You may need to go install github.com/goreleaser/goreleaser/[email protected]7.1 first
GORELEASER="goreleaser build --clean"
if [ -z "$CI" ] || [[ "${GITHUB_REF_NAME:-}" == draft-* ]]; then
GORELEASER+=" --snapshot"
snap/snapcraft.yaml
+2 -2
View File
@@ -1,5 +1,5 @@
name: yq
version: 'v4.53.3'
version: 'v4.53.6'
summary: A lightweight and portable command-line data file processor
description: |
`yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files.
@@ -32,6 +32,6 @@ parts:
build-environment:
- CGO_ENABLED: 0
source: https://github.com/mikefarah/yq.git
source-tag: v4.53.3
source-tag: v4.53.6
build-snaps:
- go/latest/stable
test/format_test.go
+1 -1
View File
@@ -17,7 +17,7 @@ func TestFormatStringFromFilename(t *testing.T) {
{"FILE.JSON", "json"},
{"file.properties", "properties"},
{"file.xml", "xml"},
{"file.unknown", "unknown"},
{"file.unknown", "yaml"},
// filenames without extensions
{"file", "yaml"},