From 8d5a5cf41b50b5225adbcc825c6faa857cea80b7 Mon Sep 17 00:00:00 2001 From: scossu Date: Mon, 22 Sep 2025 17:31:27 -0400 Subject: [PATCH 1/8] Allow encoding data with missing fields. --- ftcsv.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftcsv.lua b/ftcsv.lua index 400bef4..05d9142 100644 --- a/ftcsv.lua +++ b/ftcsv.lua @@ -814,7 +814,7 @@ local function initializeGenerator(inputTable, delimiter, options) if headers == nil then headers = extractHeadersFromTable(inputTable) end - validateHeaders(headers, inputTable) + if not options.allowEmpty then validateHeaders(headers, inputTable) end local escapedHeaders = escapeHeadersForOutput(headers, delimiter, options) local output = initializeOutputWithEscapedHeaders(escapedHeaders, delimiter, options) From f2d491ed9d5b151c160637e6df3cb7fd21e5380c Mon Sep 17 00:00:00 2001 From: scossu Date: Mon, 22 Sep 2025 17:40:45 -0400 Subject: [PATCH 2/8] Add allowEmpty documentation to README. --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 1b9e3be..93393b8 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,25 @@ file:close() local output = ftcsv.encode(everyUser, {encodeNilAs=0}) -- for setting it to 0 ``` +- `allowEmpty` + If set to a non-`nil` value, this option allows encoding data sets that are entirely missing a field that was specified in `fieldsToKeep`. Otherwise, ftcsv would raise an error. + + ```lua + local data = { + {a = 1, b = 2, c = 3}, + {a = 10, b = 20}, + {a = 100, c = 200}, + } + ftcsv.encode(data, {fieldsToKeep = {"a", "b", "c", "d"}}) --> [throws an error] + ftcsv.encode(data, {fieldsToKeep = {"a", "b", "c", "d"}, allowEmpty = true}) + --> [[ + --> "a","b","c","d" + --> "1","2","3","nil" + --> "10","20","nil","nil" + --> "100","nil","200","nil" + --> ]] + ``` ## Error Handling ftcsv returns a litany of errors when passed a bad csv file or incorrect parameters. You can find a more detailed explanation of the more cryptic errors in [ERRORS.md](ERRORS.md) From 2452959ad6592a590dfd1422609f7a5f3e057445 Mon Sep 17 00:00:00 2001 From: scossu Date: Mon, 22 Sep 2025 17:53:17 -0400 Subject: [PATCH 3/8] Adapt to option check syntax used elsewhere. --- ftcsv.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ftcsv.lua b/ftcsv.lua index 05d9142..ce95065 100644 --- a/ftcsv.lua +++ b/ftcsv.lua @@ -814,7 +814,9 @@ local function initializeGenerator(inputTable, delimiter, options) if headers == nil then headers = extractHeadersFromTable(inputTable) end - if not options.allowEmpty then validateHeaders(headers, inputTable) end + if options and options.allowEmpty == nil then + validateHeaders(headers, inputTable) + end local escapedHeaders = escapeHeadersForOutput(headers, delimiter, options) local output = initializeOutputWithEscapedHeaders(escapedHeaders, delimiter, options) From 8eec2494d1c2f94e05547eb6869ce797fd20cc8a Mon Sep 17 00:00:00 2001 From: scossu Date: Thu, 25 Sep 2025 17:06:10 -0400 Subject: [PATCH 4/8] Rename `allowEmpty` to `allowMissingKeys`; add unit test. --- README.md | 4 ++-- ftcsv.lua | 9 ++++++--- spec/csvs/missing_keys.csv | 4 ++++ spec/json/missing_keys.json | 17 +++++++++++++++++ spec/parse_encode_spec.lua | 14 ++++++++++++++ 5 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 spec/csvs/missing_keys.csv create mode 100644 spec/json/missing_keys.json diff --git a/README.md b/README.md index 93393b8..59db1ed 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ file:close() local output = ftcsv.encode(everyUser, {encodeNilAs=0}) -- for setting it to 0 ``` -- `allowEmpty` +- `allowMissingKeys` If set to a non-`nil` value, this option allows encoding data sets that are entirely missing a field that was specified in `fieldsToKeep`. Otherwise, ftcsv would raise an error. @@ -191,7 +191,7 @@ file:close() {a = 100, c = 200}, } ftcsv.encode(data, {fieldsToKeep = {"a", "b", "c", "d"}}) --> [throws an error] - ftcsv.encode(data, {fieldsToKeep = {"a", "b", "c", "d"}, allowEmpty = true}) + ftcsv.encode(data, {fieldsToKeep = {"a", "b", "c", "d"}, allowMissingKeys = true}) --> [[ --> "a","b","c","d" --> "1","2","3","nil" diff --git a/ftcsv.lua b/ftcsv.lua index ce95065..46d4865 100644 --- a/ftcsv.lua +++ b/ftcsv.lua @@ -814,7 +814,7 @@ local function initializeGenerator(inputTable, delimiter, options) if headers == nil then headers = extractHeadersFromTable(inputTable) end - if options and options.allowEmpty == nil then + if options and options.allowMissingKeys == nil then validateHeaders(headers, inputTable) end @@ -828,8 +828,11 @@ function ftcsv.encode(inputTable, delimiter, options) local delimiter, options = determineArgumentOrder(delimiter, options) local output, headers = initializeGenerator(inputTable, delimiter, options) - for i, line in csvLineGenerator(inputTable, delimiter, headers, options) do - output[i+1] = line + if options.noHeader then output = {} end + if not options.noData then + for i, line in csvLineGenerator(inputTable, delimiter, headers, options) do + output[i+1] = line + end end -- combine and return final string diff --git a/spec/csvs/missing_keys.csv b/spec/csvs/missing_keys.csv new file mode 100644 index 0000000..9d0480c --- /dev/null +++ b/spec/csvs/missing_keys.csv @@ -0,0 +1,4 @@ +a,b,c +1,2,3 +10,20, +100,,300 diff --git a/spec/json/missing_keys.json b/spec/json/missing_keys.json new file mode 100644 index 0000000..446adcd --- /dev/null +++ b/spec/json/missing_keys.json @@ -0,0 +1,17 @@ +[ + { + "a": 1, + "b": 2, + "c": 3 + }, + { + "a": 10, + "b": 20, + "c": "nil" + }, + { + "a": 100, + "b": "nil", + "c": 300 + } +] diff --git a/spec/parse_encode_spec.lua b/spec/parse_encode_spec.lua index 0f37d6f..7b766f3 100644 --- a/spec/parse_encode_spec.lua +++ b/spec/parse_encode_spec.lua @@ -114,3 +114,17 @@ describe("csv encode without quotes", function() end) end end) + +describe("csv encode with missing keys", function() + it("should handle missing_keys", function() + local jsonFile = loadFile("spec/json/missing_keys.json") + local jsonDecode = cjson.decode(jsonFile) + local reEncoded = ftcsv.parse(ftcsv.encode( + jsonDecode, ",", { + fieldsToKeep = {"a", "b", "c", "d"}, + allowMissingKeys = true, + } + ), ",", {loadFromString=true}) + assert.are.same(jsonDecode, reEncoded) + end) +end) From 07a16ee314552f205cf7c5d8716a2a35f7e93ae0 Mon Sep 17 00:00:00 2001 From: scossu Date: Thu, 25 Sep 2025 17:18:10 -0400 Subject: [PATCH 5/8] Add commented out full test (it breaks). --- spec/parse_encode_spec.lua | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/spec/parse_encode_spec.lua b/spec/parse_encode_spec.lua index 7b766f3..b0c72bb 100644 --- a/spec/parse_encode_spec.lua +++ b/spec/parse_encode_spec.lua @@ -115,16 +115,34 @@ describe("csv encode without quotes", function() end end) +--[[ This breaks simple_crlf. +describe("csv encode with missing keys", function() + for _, value in ipairs(files) do + it("should handle " .. value, function() + local jsonFile = loadFile("spec/json/" .. value .. ".json") + local jsonDecode = cjson.decode(jsonFile) + local reEncoded = ftcsv.parse(ftcsv.encode( + jsonDecode, ",", { + fieldsToKeep = {"a", "b", "c", "d"}, + allowMissingKeys = true, + } + ), ",", {loadFromString=true}) + assert.are.same(jsonDecode, reEncoded) + end) + end +end) +--]] + describe("csv encode with missing keys", function() it("should handle missing_keys", function() local jsonFile = loadFile("spec/json/missing_keys.json") local jsonDecode = cjson.decode(jsonFile) local reEncoded = ftcsv.parse(ftcsv.encode( - jsonDecode, ",", { - fieldsToKeep = {"a", "b", "c", "d"}, - allowMissingKeys = true, - } - ), ",", {loadFromString=true}) + jsonDecode, ",", { + fieldsToKeep = {"a", "b", "c", "d"}, + allowEmptyKeys = true, + } + ), ",", {loadFromString=true}) assert.are.same(jsonDecode, reEncoded) end) end) From 6df872f639b22952bb18ec64ed890b69ea346d65 Mon Sep 17 00:00:00 2001 From: scossu Date: Sun, 28 Sep 2025 22:01:44 -0400 Subject: [PATCH 6/8] Remove extraneous code. --- ftcsv.lua | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ftcsv.lua b/ftcsv.lua index 46d4865..4d3b869 100644 --- a/ftcsv.lua +++ b/ftcsv.lua @@ -828,13 +828,6 @@ function ftcsv.encode(inputTable, delimiter, options) local delimiter, options = determineArgumentOrder(delimiter, options) local output, headers = initializeGenerator(inputTable, delimiter, options) - if options.noHeader then output = {} end - if not options.noData then - for i, line in csvLineGenerator(inputTable, delimiter, headers, options) do - output[i+1] = line - end - end - -- combine and return final string return table.concat(output) end From 0d2d2b9d187e9ecff1c4c8949606102d30b54f95 Mon Sep 17 00:00:00 2001 From: scossu Date: Sun, 28 Sep 2025 22:03:47 -0400 Subject: [PATCH 7/8] Re-add original code. --- ftcsv.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ftcsv.lua b/ftcsv.lua index 4d3b869..5a0690d 100644 --- a/ftcsv.lua +++ b/ftcsv.lua @@ -828,6 +828,10 @@ function ftcsv.encode(inputTable, delimiter, options) local delimiter, options = determineArgumentOrder(delimiter, options) local output, headers = initializeGenerator(inputTable, delimiter, options) + for i, line in csvLineGenerator(inputTable, delimiter, headers, options) do + output[i+1] = line + end + -- combine and return final string return table.concat(output) end From 92adf9a6deb51ae06399cc05fa03ed26cfc80c0f Mon Sep 17 00:00:00 2001 From: scossu Date: Mon, 29 Sep 2025 08:24:39 -0400 Subject: [PATCH 8/8] Fix tests. --- spec/json/missing_keys.json | 21 +++++++++++++-------- spec/parse_encode_spec.lua | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/spec/json/missing_keys.json b/spec/json/missing_keys.json index 446adcd..d22ad49 100644 --- a/spec/json/missing_keys.json +++ b/spec/json/missing_keys.json @@ -1,17 +1,22 @@ [ { - "a": 1, - "b": 2, - "c": 3 + "a": "1", + "b": "2", + "c": "3", + "d": "nil" }, { - "a": 10, - "b": 20, - "c": "nil" + "a": "10", + "b": "20", + "c": "nil", + "d": "nil" + }, { - "a": 100, + "a": "100", "b": "nil", - "c": 300 + "c": "300", + "d": "nil" + } ] diff --git a/spec/parse_encode_spec.lua b/spec/parse_encode_spec.lua index b0c72bb..969c6bb 100644 --- a/spec/parse_encode_spec.lua +++ b/spec/parse_encode_spec.lua @@ -140,7 +140,7 @@ describe("csv encode with missing keys", function() local reEncoded = ftcsv.parse(ftcsv.encode( jsonDecode, ",", { fieldsToKeep = {"a", "b", "c", "d"}, - allowEmptyKeys = true, + allowMissingKeys = true, } ), ",", {loadFromString=true}) assert.are.same(jsonDecode, reEncoded)