From 78dc444c15ef09484da68185c41d54b985c447a6 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:33:54 +1000 Subject: [PATCH] Fix INI parser splitting keys containing a colon (e.g. `this:key = value`) (#2886) * Initial plan * Add --ini-key-value-delimiters flag to fix colon-in-key INI parsing Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> * Generate separate ini.md doc with its own header for INI scenarios Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> * Clarify default colon-splitting scenario doc as pre-fix behavior Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- cmd/root.go | 1 + pkg/yqlib/decoder_ini.go | 1 + pkg/yqlib/doc/operators/system-operators.md | 4 +- pkg/yqlib/doc/usage/headers/ini.md | 7 ++ pkg/yqlib/doc/usage/ini.md | 111 ++++++++++++++++++++ pkg/yqlib/ini.go | 3 + pkg/yqlib/ini_test.go | 80 +++++++++++++- 7 files changed, 201 insertions(+), 6 deletions(-) create mode 100644 pkg/yqlib/doc/usage/headers/ini.md create mode 100644 pkg/yqlib/doc/usage/ini.md diff --git a/cmd/root.go b/cmd/root.go index a7cf7354..297f4907 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -157,6 +157,7 @@ yq -P -oy sample.json rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredINIPreferences.PreserveSurroundedQuote, "ini-preserve-quotes", yqlib.ConfiguredINIPreferences.PreserveSurroundedQuote, "preserve surrounding quotes on INI values during round-trip") + rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredINIPreferences.KeyValueDelimiters, "ini-key-value-delimiters", yqlib.ConfiguredINIPreferences.KeyValueDelimiters, "sequence of delimiters used to separate keys and values when parsing INI") rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)") diff --git a/pkg/yqlib/decoder_ini.go b/pkg/yqlib/decoder_ini.go index 2c1a5db8..1c4aefe7 100644 --- a/pkg/yqlib/decoder_ini.go +++ b/pkg/yqlib/decoder_ini.go @@ -44,6 +44,7 @@ func (dec *iniDecoder) Decode() (*CandidateNode, error) { // Parse the INI content loadOpts := ini.LoadOptions{ PreserveSurroundedQuote: dec.prefs.PreserveSurroundedQuote, + KeyValueDelimiters: dec.prefs.KeyValueDelimiters, } cfg, err := ini.LoadSources(loadOpts, content) if err != nil { diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md index de0a8caf..df1a76cd 100644 --- a/pkg/yqlib/doc/operators/system-operators.md +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -51,7 +51,7 @@ country: Australia ``` then ```bash -yq --security-enable-system-operator '.country = system("/bin/echo"; "test")' sample.yml +yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml ``` will output ```yaml @@ -67,7 +67,7 @@ a: hello ``` then ```bash -yq --security-enable-system-operator '.a = system("/bin/echo")' sample.yml +yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml ``` will output ```yaml diff --git a/pkg/yqlib/doc/usage/headers/ini.md b/pkg/yqlib/doc/usage/headers/ini.md new file mode 100644 index 00000000..bfd60100 --- /dev/null +++ b/pkg/yqlib/doc/usage/headers/ini.md @@ -0,0 +1,7 @@ +# INI + +Encode and decode to and from INI. Sections become maps, and keys without a section are added to the top level. + +By default, both `=` and `:` are treated as key/value delimiters (matching the underlying INI parser's defaults). This can be controlled with the `--ini-key-value-delimiters` flag, which is useful when keys legitimately contain a colon, e.g. `this:key = value`. + +See below for examples diff --git a/pkg/yqlib/doc/usage/ini.md b/pkg/yqlib/doc/usage/ini.md new file mode 100644 index 00000000..3961db31 --- /dev/null +++ b/pkg/yqlib/doc/usage/ini.md @@ -0,0 +1,111 @@ +# INI + +Encode and decode to and from INI. Sections become maps, and keys without a section are added to the top level. + +By default, both `=` and `:` are treated as key/value delimiters (matching the underlying INI parser's defaults). This can be controlled with the `--ini-key-value-delimiters` flag, which is useful when keys legitimately contain a colon, e.g. `this:key = value`. + +See below for examples + +## Parse INI: simple +Given a sample.ini file of: +```ini +[section] +key = value + +``` +then +```bash +yq -p=ini sample.ini +``` +will output +```yaml +section: + key: value +``` + +## Encode INI: simple +Given a sample.yml file of: +```yaml +section: {key: value} +``` +then +```bash +yq -o=ini '.' sample.yml +``` +will output +```ini +[section] +key = value +``` + +## Roundtrip INI: simple +Given a sample.ini file of: +```ini +[section] +key = value + +``` +then +```bash +yq -p=ini -o=ini sample.ini +``` +will output +```ini +[section] +key = value +``` + +## bad ini +Given a sample.ini file of: +```ini +[section\nkey = value +``` +then an error is expected: +``` +bad file 'sample.yml': failed to parse INI content: unclosed section: [section\nkey = value +``` + +## Parse INI: key with colon +By default, the key/value delimiters are "=:", so ':' is treated the same as '=' and the key is split at the first delimiter found. See the next example for how to avoid this using `--ini-key-value-delimiters`. + +Given a sample.ini file of: +```ini +[this:section] +this:line = should really work + +``` +then +```bash +yq -p=ini sample.ini +``` +will output +```yaml +this:section: + this: line = should really work +``` + +## Parse INI: key with colon, using --ini-key-value-delimiters +Keys containing a colon (e.g. Mercurial config files) are parsed correctly when ':' is removed from the key/value delimiters. + +Given a sample.ini file of: +```ini +[this:section] +this:line = should really work + +``` +then +```bash +yq -p=ini --ini-key-value-delimiters='=' sample.ini +``` +will output +```yaml +this:section: + this:line: should really work +``` + +instead of +```yaml +this:section: + this: line = should really work +``` + diff --git a/pkg/yqlib/ini.go b/pkg/yqlib/ini.go index d5bf419d..91a8d47a 100644 --- a/pkg/yqlib/ini.go +++ b/pkg/yqlib/ini.go @@ -3,12 +3,14 @@ package yqlib type INIPreferences struct { ColorsEnabled bool PreserveSurroundedQuote bool + KeyValueDelimiters string } func NewDefaultINIPreferences() INIPreferences { return INIPreferences{ ColorsEnabled: false, PreserveSurroundedQuote: false, + KeyValueDelimiters: "=:", } } @@ -16,6 +18,7 @@ func (p *INIPreferences) Copy() INIPreferences { return INIPreferences{ ColorsEnabled: p.ColorsEnabled, PreserveSurroundedQuote: p.PreserveSurroundedQuote, + KeyValueDelimiters: p.KeyValueDelimiters, } } diff --git a/pkg/yqlib/ini_test.go b/pkg/yqlib/ini_test.go index f53bbb5f..9a6d8cf5 100644 --- a/pkg/yqlib/ini_test.go +++ b/pkg/yqlib/ini_test.go @@ -33,6 +33,23 @@ color_theme = "Default" theme_background = "False" ` +const colonKeyINIInput = `[this:section] +this:line = should really work +` + +// expectedColonKeyDefaultYaml demonstrates the default "=:" key/value delimiters +// splitting "this:line" into key "this" with value "line = should really work", +// since ":" is treated the same as "=" by default. +const expectedColonKeyDefaultYaml = `this:section: + this: line = should really work +` + +// expectedColonKeyEqualsOnlyYaml shows the fix: with --ini-key-value-delimiters +// set to "=" only, "this:line" is kept intact as the key. +const expectedColonKeyEqualsOnlyYaml = `this:section: + this:line: should really work +` + var iniScenarios = []formatScenario{ { description: "Parse INI: simple", @@ -58,6 +75,13 @@ var iniScenarios = []formatScenario{ expectedError: `bad file 'sample.yml': failed to parse INI content: unclosed section: [section\nkey = value`, scenarioType: "decode-error", }, + { + description: "Parse INI: key with colon", + subdescription: fmt.Sprintf("By default, the key/value delimiters are %q, so ':' is treated the same as '=' and the key is split at the first delimiter found. See the next example for how to avoid this using `--ini-key-value-delimiters`.", ConfiguredINIPreferences.KeyValueDelimiters), + input: colonKeyINIInput, + expected: expectedColonKeyDefaultYaml, + scenarioType: "decode", + }, } // iniPreserveQuotesPrefs returns INIPreferences with PreserveSurroundedQuote enabled. @@ -76,6 +100,24 @@ var iniPreserveQuotesScenarios = []formatScenario{ }, } +// iniEqualsOnlyKeyValueDelimiterPrefs returns INIPreferences with KeyValueDelimiters +// set to "=" only, so that ":" is not treated as a key/value separator. +func iniEqualsOnlyKeyValueDelimiterPrefs() INIPreferences { + prefs := NewDefaultINIPreferences() + prefs.KeyValueDelimiters = "=" + return prefs +} + +var iniKeyValueDelimitersScenarios = []formatScenario{ + { + description: "Parse INI: key with colon, using --ini-key-value-delimiters", + subdescription: "Keys containing a colon (e.g. Mercurial config files) are parsed correctly when ':' is removed from the key/value delimiters.", + input: colonKeyINIInput, + expected: expectedColonKeyEqualsOnlyYaml, + scenarioType: "decode-key-value-delimiters", + }, +} + func documentRoundtripINIScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) @@ -139,6 +181,8 @@ func testINIScenario(t *testing.T, s formatScenario) { } else { test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description) } + case "decode-key-value-delimiters": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(iniEqualsOnlyKeyValueDelimiterPrefs()), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } @@ -158,6 +202,8 @@ func documentINIScenario(_ *testing.T, w *bufio.Writer, i interface{}) { documentRoundtripINIScenario(w, s) case "decode-error": documentDecodeErrorINIScenario(w, s) + case "decode-key-value-delimiters": + documentDecodeKeyValueDelimitersINIScenario(w, s) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } @@ -202,6 +248,26 @@ func documentDecodeErrorINIScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("```\n%v\n```\n\n", s.expectedError)) } +func documentDecodeKeyValueDelimitersINIScenario(w *bufio.Writer, s formatScenario) { + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + writeOrPanic(w, "Given a sample.ini file of:\n") + writeOrPanic(w, fmt.Sprintf("```ini\n%v\n```\n", s.input)) + + writeOrPanic(w, "then\n") + writeOrPanic(w, "```bash\nyq -p=ini --ini-key-value-delimiters='=' sample.ini\n```\n") + writeOrPanic(w, "will output\n") + writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(iniEqualsOnlyKeyValueDelimiterPrefs()), NewYamlEncoder(ConfiguredYamlPreferences)))) + + writeOrPanic(w, "instead of\n") + writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(NewDefaultINIPreferences()), NewYamlEncoder(ConfiguredYamlPreferences)))) +} + func TestINIDecoderInitResetsFinished(t *testing.T) { decoder := NewINIDecoder(NewDefaultINIPreferences()) firstDocuments, err := readDocuments(strings.NewReader("[first]\nkey = value\n"), "first.ini", 0, decoder) @@ -221,11 +287,17 @@ func TestINIScenarios(t *testing.T) { for _, tt := range iniScenarios { testINIScenario(t, tt) } - genericScenarios := make([]interface{}, len(iniScenarios)) - for i, s := range iniScenarios { - genericScenarios[i] = s + for _, tt := range iniKeyValueDelimitersScenarios { + testINIScenario(t, tt) } - documentScenarios(t, "usage", "convert", genericScenarios, documentINIScenario) + genericScenarios := make([]interface{}, 0, len(iniScenarios)+len(iniKeyValueDelimitersScenarios)) + for _, s := range iniScenarios { + genericScenarios = append(genericScenarios, s) + } + for _, s := range iniKeyValueDelimitersScenarios { + genericScenarios = append(genericScenarios, s) + } + documentScenarios(t, "usage", "ini", genericScenarios, documentINIScenario) } func testINIPreserveQuotesScenario(t *testing.T, s formatScenario) {