mirror of
https://github.com/mikefarah/yq.git
synced 2026-10-11 21:39:52 +00:00
Fix INI parser splitting keys containing a colon (e.g. this:key = value) (#2886)
* Initial plan * Add --ini-key-value-delimiters flag to fix colon-in-key INI parsing Co-authored-by: mikefarah <[email protected]> * Generate separate ini.md doc with its own header for INI scenarios Co-authored-by: mikefarah <[email protected]> * Clarify default colon-splitting scenario doc as pre-fix behavior Co-authored-by: mikefarah <[email protected]> --------- Co-authored-by: copilot-swe-agent[bot] <[email protected]> Co-authored-by: mikefarah <[email protected]>
This commit is contained in:
Copilot
@@ -157,6 +157,7 @@ yq -P -oy sample.json
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables")
|
||||
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredINIPreferences.PreserveSurroundedQuote, "ini-preserve-quotes", yqlib.ConfiguredINIPreferences.PreserveSurroundedQuote, "preserve surrounding quotes on INI values during round-trip")
|
||||
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredINIPreferences.KeyValueDelimiters, "ini-key-value-delimiters", yqlib.ConfiguredINIPreferences.KeyValueDelimiters, "sequence of delimiters used to separate keys and values when parsing INI")
|
||||
|
||||
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values")
|
||||
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)")
|
||||
|
||||
@@ -44,6 +44,7 @@ func (dec *iniDecoder) Decode() (*CandidateNode, error) {
|
||||
// Parse the INI content
|
||||
loadOpts := ini.LoadOptions{
|
||||
PreserveSurroundedQuote: dec.prefs.PreserveSurroundedQuote,
|
||||
KeyValueDelimiters: dec.prefs.KeyValueDelimiters,
|
||||
}
|
||||
cfg, err := ini.LoadSources(loadOpts, content)
|
||||
if err != nil {
|
||||
|
||||
@@ -51,7 +51,7 @@ country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq --security-enable-system-operator '.country = system("/bin/echo"; "test")' sample.yml
|
||||
yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
@@ -67,7 +67,7 @@ a: hello
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq --security-enable-system-operator '.a = system("/bin/echo")' sample.yml
|
||||
yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# INI
|
||||
|
||||
Encode and decode to and from INI. Sections become maps, and keys without a section are added to the top level.
|
||||
|
||||
By default, both `=` and `:` are treated as key/value delimiters (matching the underlying INI parser's defaults). This can be controlled with the `--ini-key-value-delimiters` flag, which is useful when keys legitimately contain a colon, e.g. `this:key = value`.
|
||||
|
||||
See below for examples
|
||||
@@ -0,0 +1,111 @@
|
||||
# INI
|
||||
|
||||
Encode and decode to and from INI. Sections become maps, and keys without a section are added to the top level.
|
||||
|
||||
By default, both `=` and `:` are treated as key/value delimiters (matching the underlying INI parser's defaults). This can be controlled with the `--ini-key-value-delimiters` flag, which is useful when keys legitimately contain a colon, e.g. `this:key = value`.
|
||||
|
||||
See below for examples
|
||||
|
||||
## Parse INI: simple
|
||||
Given a sample.ini file of:
|
||||
```ini
|
||||
[section]
|
||||
key = value
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq -p=ini sample.ini
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
section:
|
||||
key: value
|
||||
```
|
||||
|
||||
## Encode INI: simple
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
section: {key: value}
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq -o=ini '.' sample.yml
|
||||
```
|
||||
will output
|
||||
```ini
|
||||
[section]
|
||||
key = value
|
||||
```
|
||||
|
||||
## Roundtrip INI: simple
|
||||
Given a sample.ini file of:
|
||||
```ini
|
||||
[section]
|
||||
key = value
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq -p=ini -o=ini sample.ini
|
||||
```
|
||||
will output
|
||||
```ini
|
||||
[section]
|
||||
key = value
|
||||
```
|
||||
|
||||
## bad ini
|
||||
Given a sample.ini file of:
|
||||
```ini
|
||||
[section\nkey = value
|
||||
```
|
||||
then an error is expected:
|
||||
```
|
||||
bad file 'sample.yml': failed to parse INI content: unclosed section: [section\nkey = value
|
||||
```
|
||||
|
||||
## Parse INI: key with colon
|
||||
By default, the key/value delimiters are "=:", so ':' is treated the same as '=' and the key is split at the first delimiter found. See the next example for how to avoid this using `--ini-key-value-delimiters`.
|
||||
|
||||
Given a sample.ini file of:
|
||||
```ini
|
||||
[this:section]
|
||||
this:line = should really work
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq -p=ini sample.ini
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
this:section:
|
||||
this: line = should really work
|
||||
```
|
||||
|
||||
## Parse INI: key with colon, using --ini-key-value-delimiters
|
||||
Keys containing a colon (e.g. Mercurial config files) are parsed correctly when ':' is removed from the key/value delimiters.
|
||||
|
||||
Given a sample.ini file of:
|
||||
```ini
|
||||
[this:section]
|
||||
this:line = should really work
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq -p=ini --ini-key-value-delimiters='=' sample.ini
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
this:section:
|
||||
this:line: should really work
|
||||
```
|
||||
|
||||
instead of
|
||||
```yaml
|
||||
this:section:
|
||||
this: line = should really work
|
||||
```
|
||||
|
||||
@@ -3,12 +3,14 @@ package yqlib
|
||||
type INIPreferences struct {
|
||||
ColorsEnabled bool
|
||||
PreserveSurroundedQuote bool
|
||||
KeyValueDelimiters string
|
||||
}
|
||||
|
||||
func NewDefaultINIPreferences() INIPreferences {
|
||||
return INIPreferences{
|
||||
ColorsEnabled: false,
|
||||
PreserveSurroundedQuote: false,
|
||||
KeyValueDelimiters: "=:",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +18,7 @@ func (p *INIPreferences) Copy() INIPreferences {
|
||||
return INIPreferences{
|
||||
ColorsEnabled: p.ColorsEnabled,
|
||||
PreserveSurroundedQuote: p.PreserveSurroundedQuote,
|
||||
KeyValueDelimiters: p.KeyValueDelimiters,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+76
-4
@@ -33,6 +33,23 @@ color_theme = "Default"
|
||||
theme_background = "False"
|
||||
`
|
||||
|
||||
const colonKeyINIInput = `[this:section]
|
||||
this:line = should really work
|
||||
`
|
||||
|
||||
// expectedColonKeyDefaultYaml demonstrates the default "=:" key/value delimiters
|
||||
// splitting "this:line" into key "this" with value "line = should really work",
|
||||
// since ":" is treated the same as "=" by default.
|
||||
const expectedColonKeyDefaultYaml = `this:section:
|
||||
this: line = should really work
|
||||
`
|
||||
|
||||
// expectedColonKeyEqualsOnlyYaml shows the fix: with --ini-key-value-delimiters
|
||||
// set to "=" only, "this:line" is kept intact as the key.
|
||||
const expectedColonKeyEqualsOnlyYaml = `this:section:
|
||||
this:line: should really work
|
||||
`
|
||||
|
||||
var iniScenarios = []formatScenario{
|
||||
{
|
||||
description: "Parse INI: simple",
|
||||
@@ -58,6 +75,13 @@ var iniScenarios = []formatScenario{
|
||||
expectedError: `bad file 'sample.yml': failed to parse INI content: unclosed section: [section\nkey = value`,
|
||||
scenarioType: "decode-error",
|
||||
},
|
||||
{
|
||||
description: "Parse INI: key with colon",
|
||||
subdescription: fmt.Sprintf("By default, the key/value delimiters are %q, so ':' is treated the same as '=' and the key is split at the first delimiter found. See the next example for how to avoid this using `--ini-key-value-delimiters`.", ConfiguredINIPreferences.KeyValueDelimiters),
|
||||
input: colonKeyINIInput,
|
||||
expected: expectedColonKeyDefaultYaml,
|
||||
scenarioType: "decode",
|
||||
},
|
||||
}
|
||||
|
||||
// iniPreserveQuotesPrefs returns INIPreferences with PreserveSurroundedQuote enabled.
|
||||
@@ -76,6 +100,24 @@ var iniPreserveQuotesScenarios = []formatScenario{
|
||||
},
|
||||
}
|
||||
|
||||
// iniEqualsOnlyKeyValueDelimiterPrefs returns INIPreferences with KeyValueDelimiters
|
||||
// set to "=" only, so that ":" is not treated as a key/value separator.
|
||||
func iniEqualsOnlyKeyValueDelimiterPrefs() INIPreferences {
|
||||
prefs := NewDefaultINIPreferences()
|
||||
prefs.KeyValueDelimiters = "="
|
||||
return prefs
|
||||
}
|
||||
|
||||
var iniKeyValueDelimitersScenarios = []formatScenario{
|
||||
{
|
||||
description: "Parse INI: key with colon, using --ini-key-value-delimiters",
|
||||
subdescription: "Keys containing a colon (e.g. Mercurial config files) are parsed correctly when ':' is removed from the key/value delimiters.",
|
||||
input: colonKeyINIInput,
|
||||
expected: expectedColonKeyEqualsOnlyYaml,
|
||||
scenarioType: "decode-key-value-delimiters",
|
||||
},
|
||||
}
|
||||
|
||||
func documentRoundtripINIScenario(w *bufio.Writer, s formatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
@@ -139,6 +181,8 @@ func testINIScenario(t *testing.T, s formatScenario) {
|
||||
} else {
|
||||
test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
|
||||
}
|
||||
case "decode-key-value-delimiters":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(iniEqualsOnlyKeyValueDelimiterPrefs()), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
|
||||
default:
|
||||
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
|
||||
}
|
||||
@@ -158,6 +202,8 @@ func documentINIScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
|
||||
documentRoundtripINIScenario(w, s)
|
||||
case "decode-error":
|
||||
documentDecodeErrorINIScenario(w, s)
|
||||
case "decode-key-value-delimiters":
|
||||
documentDecodeKeyValueDelimitersINIScenario(w, s)
|
||||
default:
|
||||
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
|
||||
}
|
||||
@@ -202,6 +248,26 @@ func documentDecodeErrorINIScenario(w *bufio.Writer, s formatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("```\n%v\n```\n\n", s.expectedError))
|
||||
}
|
||||
|
||||
func documentDecodeKeyValueDelimitersINIScenario(w *bufio.Writer, s formatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
writeOrPanic(w, s.subdescription)
|
||||
writeOrPanic(w, "\n\n")
|
||||
}
|
||||
|
||||
writeOrPanic(w, "Given a sample.ini file of:\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```ini\n%v\n```\n", s.input))
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
writeOrPanic(w, "```bash\nyq -p=ini --ini-key-value-delimiters='=' sample.ini\n```\n")
|
||||
writeOrPanic(w, "will output\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(iniEqualsOnlyKeyValueDelimiterPrefs()), NewYamlEncoder(ConfiguredYamlPreferences))))
|
||||
|
||||
writeOrPanic(w, "instead of\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(NewDefaultINIPreferences()), NewYamlEncoder(ConfiguredYamlPreferences))))
|
||||
}
|
||||
|
||||
func TestINIDecoderInitResetsFinished(t *testing.T) {
|
||||
decoder := NewINIDecoder(NewDefaultINIPreferences())
|
||||
firstDocuments, err := readDocuments(strings.NewReader("[first]\nkey = value\n"), "first.ini", 0, decoder)
|
||||
@@ -221,11 +287,17 @@ func TestINIScenarios(t *testing.T) {
|
||||
for _, tt := range iniScenarios {
|
||||
testINIScenario(t, tt)
|
||||
}
|
||||
genericScenarios := make([]interface{}, len(iniScenarios))
|
||||
for i, s := range iniScenarios {
|
||||
genericScenarios[i] = s
|
||||
for _, tt := range iniKeyValueDelimitersScenarios {
|
||||
testINIScenario(t, tt)
|
||||
}
|
||||
documentScenarios(t, "usage", "convert", genericScenarios, documentINIScenario)
|
||||
genericScenarios := make([]interface{}, 0, len(iniScenarios)+len(iniKeyValueDelimitersScenarios))
|
||||
for _, s := range iniScenarios {
|
||||
genericScenarios = append(genericScenarios, s)
|
||||
}
|
||||
for _, s := range iniKeyValueDelimitersScenarios {
|
||||
genericScenarios = append(genericScenarios, s)
|
||||
}
|
||||
documentScenarios(t, "usage", "ini", genericScenarios, documentINIScenario)
|
||||
}
|
||||
|
||||
func testINIPreserveQuotesScenario(t *testing.T, s formatScenario) {
|
||||
|
||||
Reference in New Issue
Block a user