Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/insiders-features.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ CSV output applies only to tools in default toolsets whose names start with `lis

### Format

- Nested objects are flattened into dot-notation columns, for example `user.login`, `category.name`, or `head.ref`.
- Nested objects are flattened into dot-notation columns, for example `user.login`, `category.name`, or `head.ref`. A `.` or `\` inside a field name is escaped as `\.` or `\\`, so a literal `user.login` field becomes the column `user\.login` and never shares a column with a nested `user` → `login` field.
- Arrays are represented as compact single-cell values joined with `;`.
- `body` fields are whitespace-normalized so multiline Markdown does not expand a list response into many output lines.
- Response metadata present in wrapped responses, such as `pageInfo.*` and `totalCount`, is emitted as `#`-prefixed lines before the CSV rows, followed by a blank line. Tools that return a root JSON array do not include metadata preamble lines.
Expand Down
23 changes: 18 additions & 5 deletions pkg/github/csv_output.go
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ func appendFlattenedCSVFields(row map[string]string, value map[string]any, prefi
case []any:
row[column] = csvArrayValue(v)
default:
row[column] = csvColumnValue(column, v)
row[column] = csvColumnValue(key, v)
}
}
}
Expand All @@ -313,7 +313,19 @@ func csvHeaders(rows []map[string]string) []string {
return headers
}

// csvColumnKeyEscaper escapes the path separator inside a single key so that
// a literal key such as "user.login" cannot collide with the nested path
// user -> login. Escaping the escape character as well keeps the mapping
// from key paths to column names injective.
var csvColumnKeyEscaper = strings.NewReplacer(`\`, `\\`, `.`, `\.`)

// csvColumnName joins a flattened key path with ".". Within each key, "\"
// becomes "\\" and "." becomes "\.", so distinct source fields always map to
// distinct columns: {"user":{"login":...}} yields user.login, while a literal
// {"user.login":...} key yields user\.login. Keys without "." or "\" are
// unchanged.
func csvColumnName(prefix, key string) string {
key = csvColumnKeyEscaper.Replace(key)
if prefix == "" {
return key
}
Expand Down Expand Up @@ -346,9 +358,10 @@ func singleArrayKey(value map[string]any) (string, bool) {
return arrayKey, true
}

func csvColumnValue(column string, value any) string {
// csvColumnValue formats a scalar field; key is the unescaped leaf key.
func csvColumnValue(key string, value any) string {
str := scalarCSVValue(value)
if isBodyColumn(column) {
if isBodyKey(key) {
return normalizeCSVWhitespace(str)
}
return str
Expand Down Expand Up @@ -400,8 +413,8 @@ func scalarCSVValue(value any) string {
}
}

func isBodyColumn(column string) bool {
return column == "body" || strings.HasSuffix(column, ".body")
func isBodyKey(key string) bool {
return key == "body"
}

func normalizeCSVWhitespace(value string) string {
Expand Down
55 changes: 55 additions & 0 deletions pkg/github/csv_output_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -545,3 +545,58 @@ func csvRow(t *testing.T, headers []string, record []string) map[string]string {
}
return row
}

func TestJSONTextToCSVKeepsCollidingFlattenedKeysDistinct(t *testing.T) {
// Run several times: before the fix, Go map iteration order decided which
// value survived in a single shared column.
for range 20 {
csvText, err := jsonTextToCSV(`[{"user.login":"literal","user":{"login":"nested"}}]`)
require.NoError(t, err)

records := readCSVText(t, csvText)
require.Len(t, records, 2)
assert.Equal(t, []string{"user.login", `user\.login`}, records[0])

row := csvRow(t, records[0], records[1])
assert.Equal(t, "nested", row["user.login"])
assert.Equal(t, "literal", row[`user\.login`])
}
}

func TestJSONTextToCSVEscapesBackslashesInKeys(t *testing.T) {
// Without escaping "\" itself, a literal `a\.b` key would collide with the
// escaped form of a literal "a.b" key.
csvText, err := jsonTextToCSV(`[{"a.b":"dot","a\\.b":"backslash-dot","a\\":{"b":"backslash-nested"},"a":{"b":"nested"}}]`)
require.NoError(t, err)

records := readCSVText(t, csvText)
require.Len(t, records, 2)
require.Len(t, records[0], 4, "every source field gets its own column")

row := csvRow(t, records[0], records[1])
assert.Equal(t, "nested", row["a.b"])
assert.Equal(t, "dot", row[`a\.b`])
assert.Equal(t, "backslash-dot", row[`a\\\.b`])
assert.Equal(t, "backslash-nested", row[`a\\.b`])
}

func TestJSONTextToCSVKeepsCollidingMetadataKeysDistinct(t *testing.T) {
csvText, err := jsonTextToCSV(`{"pageInfo.endCursor":"literal","pageInfo":{"endCursor":"nested"},"items":[{"id":1}]}`)
require.NoError(t, err)

assert.Contains(t, csvText, "# pageInfo.endCursor: nested\n")
assert.Contains(t, csvText, "# pageInfo\\.endCursor: literal\n")
}

func TestJSONTextToCSVNormalizesOnlyBodyLeafKeys(t *testing.T) {
csvText, err := jsonTextToCSV(`[{"body":"a\n\nb","comment":{"body":"c\n d"},"x.body":"e\nf"}]`)
require.NoError(t, err)

records := readCSVText(t, csvText)
require.Len(t, records, 2)

row := csvRow(t, records[0], records[1])
assert.Equal(t, "a b", row["body"])
assert.Equal(t, "c d", row["comment.body"])
assert.Equal(t, "e\nf", row[`x\.body`], "a literal dotted key is not a nested body field")
}