From 62b30ba29aae49f0fa19c39fb9dcc580e0c6534d Mon Sep 17 00:00:00 2001 From: Pranav Tomar Date: Tue, 17 Nov 2026 00:55:00 +0530 Subject: [PATCH] fix(csv): keep colliding flattened CSV keys in distinct columns A literal dotted key such as "user.login" and the nested path user -> login both flattened to the column user.login, so one value was silently overwritten depending on map iteration order. Escape "\" and "." inside each key when building column names (user\.login for the literal key) so distinct source fields always map to distinct columns. Fixes #3865 --- docs/insiders-features.md | 2 +- pkg/github/csv_output.go | 23 +++++++++++---- pkg/github/csv_output_test.go | 55 +++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/docs/insiders-features.md b/docs/insiders-features.md index 22c8dd6433..53ad53d06d 100644 --- a/docs/insiders-features.md +++ b/docs/insiders-features.md @@ -83,7 +83,7 @@ CSV output applies only to tools in default toolsets whose names start with `lis ### Format -- Nested objects are flattened into dot-notation columns, for example `user.login`, `category.name`, or `head.ref`. +- Nested objects are flattened into dot-notation columns, for example `user.login`, `category.name`, or `head.ref`. A `.` or `\` inside a field name is escaped as `\.` or `\\`, so a literal `user.login` field becomes the column `user\.login` and never shares a column with a nested `user` → `login` field. - Arrays are represented as compact single-cell values joined with `;`. - `body` fields are whitespace-normalized so multiline Markdown does not expand a list response into many output lines. - Response metadata present in wrapped responses, such as `pageInfo.*` and `totalCount`, is emitted as `#`-prefixed lines before the CSV rows, followed by a blank line. Tools that return a root JSON array do not include metadata preamble lines. diff --git a/pkg/github/csv_output.go b/pkg/github/csv_output.go index ee4bba19db..2ed250ffa2 100644 --- a/pkg/github/csv_output.go +++ b/pkg/github/csv_output.go @@ -292,7 +292,7 @@ func appendFlattenedCSVFields(row map[string]string, value map[string]any, prefi case []any: row[column] = csvArrayValue(v) default: - row[column] = csvColumnValue(column, v) + row[column] = csvColumnValue(key, v) } } } @@ -313,7 +313,19 @@ func csvHeaders(rows []map[string]string) []string { return headers } +// csvColumnKeyEscaper escapes the path separator inside a single key so that +// a literal key such as "user.login" cannot collide with the nested path +// user -> login. Escaping the escape character as well keeps the mapping +// from key paths to column names injective. +var csvColumnKeyEscaper = strings.NewReplacer(`\`, `\\`, `.`, `\.`) + +// csvColumnName joins a flattened key path with ".". Within each key, "\" +// becomes "\\" and "." becomes "\.", so distinct source fields always map to +// distinct columns: {"user":{"login":...}} yields user.login, while a literal +// {"user.login":...} key yields user\.login. Keys without "." or "\" are +// unchanged. func csvColumnName(prefix, key string) string { + key = csvColumnKeyEscaper.Replace(key) if prefix == "" { return key } @@ -346,9 +358,10 @@ func singleArrayKey(value map[string]any) (string, bool) { return arrayKey, true } -func csvColumnValue(column string, value any) string { +// csvColumnValue formats a scalar field; key is the unescaped leaf key. +func csvColumnValue(key string, value any) string { str := scalarCSVValue(value) - if isBodyColumn(column) { + if isBodyKey(key) { return normalizeCSVWhitespace(str) } return str @@ -400,8 +413,8 @@ func scalarCSVValue(value any) string { } } -func isBodyColumn(column string) bool { - return column == "body" || strings.HasSuffix(column, ".body") +func isBodyKey(key string) bool { + return key == "body" } func normalizeCSVWhitespace(value string) string { diff --git a/pkg/github/csv_output_test.go b/pkg/github/csv_output_test.go index 0582353cc8..8e55cc6453 100644 --- a/pkg/github/csv_output_test.go +++ b/pkg/github/csv_output_test.go @@ -545,3 +545,58 @@ func csvRow(t *testing.T, headers []string, record []string) map[string]string { } return row } + +func TestJSONTextToCSVKeepsCollidingFlattenedKeysDistinct(t *testing.T) { + // Run several times: before the fix, Go map iteration order decided which + // value survived in a single shared column. + for range 20 { + csvText, err := jsonTextToCSV(`[{"user.login":"literal","user":{"login":"nested"}}]`) + require.NoError(t, err) + + records := readCSVText(t, csvText) + require.Len(t, records, 2) + assert.Equal(t, []string{"user.login", `user\.login`}, records[0]) + + row := csvRow(t, records[0], records[1]) + assert.Equal(t, "nested", row["user.login"]) + assert.Equal(t, "literal", row[`user\.login`]) + } +} + +func TestJSONTextToCSVEscapesBackslashesInKeys(t *testing.T) { + // Without escaping "\" itself, a literal `a\.b` key would collide with the + // escaped form of a literal "a.b" key. + csvText, err := jsonTextToCSV(`[{"a.b":"dot","a\\.b":"backslash-dot","a\\":{"b":"backslash-nested"},"a":{"b":"nested"}}]`) + require.NoError(t, err) + + records := readCSVText(t, csvText) + require.Len(t, records, 2) + require.Len(t, records[0], 4, "every source field gets its own column") + + row := csvRow(t, records[0], records[1]) + assert.Equal(t, "nested", row["a.b"]) + assert.Equal(t, "dot", row[`a\.b`]) + assert.Equal(t, "backslash-dot", row[`a\\\.b`]) + assert.Equal(t, "backslash-nested", row[`a\\.b`]) +} + +func TestJSONTextToCSVKeepsCollidingMetadataKeysDistinct(t *testing.T) { + csvText, err := jsonTextToCSV(`{"pageInfo.endCursor":"literal","pageInfo":{"endCursor":"nested"},"items":[{"id":1}]}`) + require.NoError(t, err) + + assert.Contains(t, csvText, "# pageInfo.endCursor: nested\n") + assert.Contains(t, csvText, "# pageInfo\\.endCursor: literal\n") +} + +func TestJSONTextToCSVNormalizesOnlyBodyLeafKeys(t *testing.T) { + csvText, err := jsonTextToCSV(`[{"body":"a\n\nb","comment":{"body":"c\n d"},"x.body":"e\nf"}]`) + require.NoError(t, err) + + records := readCSVText(t, csvText) + require.Len(t, records, 2) + + row := csvRow(t, records[0], records[1]) + assert.Equal(t, "a b", row["body"]) + assert.Equal(t, "c d", row["comment.body"]) + assert.Equal(t, "e\nf", row[`x\.body`], "a literal dotted key is not a nested body field") +}