fix: results and profile CSVs quote the reader note instead of an unvalidated clause

Sources/ImagoCore/ProfileCSV.swift

@@ -9,7 +9,7 @@
 //  axes' unit as the FILE states it (a Velox metre axis stays in metres), `px` when the image is not calibrated; `shown`
 //  only adds a line saying what the app shows them in. HEADER values pass `CSVText.headerValue`, the column header
 //  `CSVText.quoted`. The title, the validation sentence and the lineage sentence are the caller's (the kit authors no claim);
-//  when a note of the source says "unvalidated" (any case) `TidyResultsCSV.unvalidatedClause` follows the validation.
+//  when the source has reader notes, `; reader note: ` and the notes (joined by `; `) follow the validation.
 //
 //  UTF-8, LF line ends, tested byte for byte. Same lines, in the same order, as macTEM's former app copy.
 //
@@ -33,7 +33,7 @@ public nonisolated enum ProfileCSV {
     ///   - appName: names the app in the `# shown in <appName> as:` line ("macTEM").
     ///   - start, end: the line's ends in image pixels (x column, y row, pixel centres at integers).
     ///   - source: the file and its calibration as the file states them.
-    ///   - validation: the validation sentence (the clause for an unvalidated source is added).
+    ///   - validation: the validation sentence (the source's reader notes are quoted after it).
     ///   - method: the `# method:` sentence; `method(width:)` for order 1, `reflect`, mean.
     ///   - lineage: the `# lineage:` sentence.
     ///   - shown: the unit and factor the app shows positions in when it differs from the file's; nil when the same.
@@ -42,7 +42,6 @@ public nonisolated enum ProfileCSV {
                             lineage: String = noLineage, shown: (unit: String, factor: Double)? = nil) -> String? {
         guard let length = profile.positions.last, !profile.samples.isEmpty else { return nil }
         let unit = CSVText.headerValue(profile.unit)
-        let unvalidated = source.notes.contains { $0.lowercased().contains("unvalidated") }
         let notes = source.notes.isEmpty ? "none" : source.notes.map(CSVText.headerValue).joined(separator: "; ")
 
         var header: [String] = []
@@ -65,8 +64,7 @@ public nonisolated enum ProfileCSV {
         }
         header.append("# method: \(CSVText.headerValue(method ?? Self.method(width: profile.width)))")
         header.append("# notes: \(notes)")
-        header.append("# validation: \(CSVText.headerValue(validation))"
-                      + (unvalidated ? TidyResultsCSV.unvalidatedClause : ""))
+        header.append("# validation: \(CSVText.headerValue(validation + TidyResultsCSV.readerNoteClause(source.notes)))")
         header.append("# lineage: \(CSVText.headerValue(lineage))")
         header.append("# numbers: %.17g (exact round trip); nan where a NaN pixel enters a sample")
         header.append(CSVText.quoted("position (\(unit))") + ",value")

Sources/ImagoCore/TidyResultsCSV.swift

@@ -9,18 +9,27 @@
 //  cell and its words in `text`. Text cells and header values go through `CSVText.quoted` and `CSVText.headerValue`.
 //
 //  VALIDATION: the `validation` column holds the product's own sentence, and the header holds one `# validation (<product>):`
-//  line for each product present, in the order `validations` lists them. When a note of the source says "unvalidated" (any
-//  case), `unvalidatedClause` follows every sentence. The kit authors no claim: the caller supplies the sentences.
+//  line for each product present, in the order `validations` lists them. When the source has reader notes, every sentence is
+//  followed by `; reader note: ` and the notes as the reader wrote them, joined by `; `, so the file states what the reader
+//  said about its own numbers. The kit authors no claim: the caller supplies the sentences, the reader the notes.
 //
 //  Pure text: exact, tested byte for byte. UTF-8, LF line ends.
 //
 
 import Foundation
 
 public nonisolated enum TidyResultsCSV {
-    /// Added to every validation sentence when the source's notes say it is unvalidated.
+    /// No writer uses this any more: the CSVs quote the reader notes after each validation sentence instead.
+    @available(*, deprecated, message: "The CSV writers now quote the reader notes after each validation sentence; this clause is no longer written.")
     public static let unvalidatedClause = ", unvalidated (reader note)"
 
+    /// What follows every validation sentence: `; reader note: <notes joined by "; ">`, or nothing without a note. Each note
+    /// passes `CSVText.headerValue` (so a cell keeps one line), and blank notes are left out.
+    static func readerNoteClause(_ notes: [String]) -> String {
+        let said = notes.map(CSVText.headerValue).filter { !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }
+        return said.isEmpty ? "" : "; reader note: " + said.joined(separator: "; ")
+    }
+
     public static let columns = ["group", "id", "label", "quantity", "value", "unit", "text", "validation", "lineage"]
 
     /// The file and its calibration as the file states them.
@@ -103,7 +112,7 @@ public nonisolated enum TidyResultsCSV {
         lines.append("# session: \(CSVText.headerValue(session))")
         lines.append("# notes: \(notes)")
         lines.append(contentsOf: descriptionLines)
-        let flagged = source.notes.contains { $0.lowercased().contains("unvalidated") } ? unvalidatedClause : ""
+        let flagged = readerNoteClause(source.notes)
         let present = Set(rows.map(\.product))
         var written = Set<String>()
         for v in validations where present.contains(v.product) && written.insert(v.product).inserted {

Sources/ImagoFilters/GeometricPhaseAnalysis.swift

@@ -48,10 +48,10 @@ public nonisolated enum GeometricPhaseAnalysis {
 
     /// Full-size, row-major, `rows * columns` arrays.
     ///
-    /// `validation` is "none": the result agrees with planted fields (tools/gpa/planted.py) and has not been checked on a
-    /// dataset with truth, so an app must show it as unvalidated until one passes.
+    /// Tested against planted displacement fields (tools/gpa/planted.py); not yet checked on a recorded dataset with known
+    /// strain. An app that exports it names that method and reference. `validation` keeps the value "none" for that state.
     public nonisolated struct Result: Sendable {
-        /// The badge every result carries until a dataset with truth passes.
+        /// The value of `validation` while the only check is the planted fields.
         public static let validationNone = "none"
         public let validation: String = "none"
         public var rows: Int
@@ -209,7 +209,7 @@ public nonisolated enum GeometricPhaseAnalysis {
     /// plane g_col > 0 (or g_col = 0 and g_row > 0), the stronger first. The spectrum is taken after the mean is removed and a
     /// Hann window applied; each peak is refined by a parabola through the log magnitudes of the three bins on each axis.
     /// Peaks within `minRadius` cycles per pixel of the origin are skipped; the second peak must be at least 3 bins from the
-    /// first with |sin angle| >= 0.2. Unvalidated beyond the planted images.
+    /// first with |sin angle| >= 0.2. Tested on the planted images only.
     public static func findReflections(_ image: [Double], rows: Int, columns: Int, minRadius: Double = 0.02) throws
         -> (g1: (row: Double, column: Double), g2: (row: Double, column: Double)) {
         typealias E = GeometricPhaseAnalysisError

Tests/ImagoCoreTests/CSVTextTests.swift

@@ -94,6 +94,33 @@ private struct TidyPins: Decodable, Sendable {
         #expect(written >= 12)
     }
 
+    /// A source with reader notes: every validation sentence (header line and cell) is followed by the notes as the reader
+    /// wrote them, and nothing is called unvalidated. Without notes the sentence stands alone.
+    @Test func tidyQuotesTheReaderNoteAfterEachValidation() throws {
+        let row = TidyResultsCSV.Row(group: "g", id: "i", label: "l", product: "known", entries: [TidyResultsCSV.Entry("d", 1)])
+        func text(_ notes: [String]) throws -> String {
+            let source = TidyResultsCSV.Source(software: "s", fileName: "f", object: "o", width: 1, height: 1, x: nil, y: nil,
+                                               notes: notes)
+            return try #require(try TidyResultsCSV.text([row], source: source, title: "t", session: "s", descriptionLines: [],
+                                                        validations: [(product: "known", sentence: "closed form")],
+                                                        lineageLine: "# lineage: none"))
+        }
+        let noted = try text(["scale from the tag", "UNVALIDATED"])
+        let lines = noted.components(separatedBy: "\n")
+        #expect(lines.contains("# validation (known): closed form; reader note: scale from the tag; UNVALIDATED"))
+        #expect(lines.contains("g,i,l,d,1,,,closed form; reader note: scale from the tag; UNVALIDATED,"))
+        #expect(!noted.contains("unvalidated"))     // the reader's own word is quoted, never a lower-case verdict
+        let plain = try text(["origin from the header"])
+        #expect(plain.contains("closed form; reader note: origin from the header"))
+        #expect(!plain.lowercased().contains("unvalidated"))
+        let bare = try text([])
+        #expect(bare.contains("# validation (known): closed form\n") && bare.hasSuffix("g,i,l,d,1,,,closed form,\n"))
+        // a cell keeps one line (the note passes headerValue) and a blank note adds nothing
+        let multi = try text(["cr\r\nlf", "", " "])
+        #expect(multi.hasSuffix("g,i,l,d,1,,,closed form; reader note: cr  lf,\n"))
+        #expect(try text(["", "\n"]) == bare.replacingOccurrences(of: "# notes: none", with: "# notes: ;  "))
+    }
+
     @Test func tidyRefusesAProductWithoutASentenceAndWritesNothingForNoRow() throws {
         let source = TidyResultsCSV.Source(software: "s", fileName: "f", object: "o", width: 1, height: 1, x: nil, y: nil, notes: [])
         let row = TidyResultsCSV.Row(group: "g", id: "i", label: "l", product: "mystery", entries: [])

Tests/ImagoCoreTests/ProfileCSVTests.swift

@@ -1,5 +1,5 @@
 // ProfileCSV: the whole text of a planted profile byte for byte (the lines and order macTEM's app copy wrote), the shown
-// line, the unvalidated clause, header sanitising, nan, and an empty profile that is not a file. Text is tested exact.
+// line, the reader note after the validation, header sanitising, nan, and an empty profile that is not a file. Text is tested exact.
 import Foundation
 import ImagoCore
 import Testing
@@ -53,13 +53,14 @@ import Testing
         #expect(try planted() == expected)
     }
 
-    @Test func shownLineAndUnvalidatedClauseAndSanitising() throws {
-        let text = try #require(try planted(notes: ["scale UNVALIDATED\r\nnext"], fileName: "a\u{2028}b.dm4",
+    @Test func shownLineAndReaderNoteAndSanitising() throws {
+        let text = try #require(try planted(notes: ["scale from the tag\r\nnext", "second"], fileName: "a\u{2028}b.dm4",
                                             shown: (unit: "nm", factor: 1e9)))
         let lines = text.components(separatedBy: "\n")
         #expect(lines.contains("# shown in macTEM as: nm (x 1000000000.0)"))
-        #expect(lines.contains("# notes: scale UNVALIDATED  next"))
-        #expect(lines.contains("# validation: " + validation + ", unvalidated (reader note)"))
+        #expect(lines.contains("# notes: scale from the tag  next; second"))
+        #expect(lines.contains("# validation: " + validation + "; reader note: scale from the tag  next; second"))
+        #expect(!text.lowercased().contains("unvalidated"))
         #expect(lines.contains("# file: a b.dm4"))
         let keys = lines.prefix { $0.hasPrefix("#") }.compactMap { $0.dropFirst(2).components(separatedBy: ": ").first }
         #expect(keys.contains("shown in macTEM as"))

tools/consumer-check/Sources/consumer-check/main.swift

@@ -402,7 +402,7 @@ check(foundSet == [[0, 0.125], [0.125, 0]], "public GPA finds the two reflection
 let gpa = try GeometricPhaseAnalysis.analyze(lattice, rows: 64, columns: 64, g1: (0, 0.125), g2: (0.125, 0))
 let worst = (gpa.strainRR + gpa.strainCC + gpa.strainRC + gpa.rotation).map(abs).max() ?? .nan
 check(gpa.validation == GeometricPhaseAnalysis.Result.validationNone && worst < 1e-8,
-      "public GPA of a perfect lattice: zero strain and rotation, badged unvalidated (worst \(worst))")
+      "public GPA of a perfect lattice: zero strain and rotation, validation value none (worst \(worst))")
 do { _ = try GeometricPhaseAnalysis.analyze(lattice, rows: 64, columns: 64, g1: (0, 0.125), g2: (0, 0.25)); check(false, "collinear g") }
 catch let e as GeometricPhaseAnalysisError { check(e == .collinearReflections, "public GPA refuses collinear reflections") }
 catch { check(false, "collinear g: another error \(error)") }