diff --git a/Sources/SpectroEDX/Quant/FitRange.swift b/Sources/SpectroEDX/Quant/FitRange.swift index 55f878c..e2049bf 100644 --- a/Sources/SpectroEDX/Quant/FitRange.swift +++ b/Sources/SpectroEDX/Quant/FitRange.swift @@ -169,7 +169,7 @@ public nonisolated enum FitRangeSensitivityCheck { var refit = input refit.method.fitToKeV = to refit.refinement = q.refinement - refit.sumEscape = PooledQuantifier.sameSumEscape(as: q) // the reported fit's sum-peak decision, not a new one + refit = PooledQuantifier.pinningReportedChoices(refit, as: q) // the reported fit's sum-peak decision and dead layer, not new ones let after: PooledQuantification do { after = try PooledQuantifier.run(refit, tables: nil, cancelled: cancelled, progress: progress.map { p in { @Sendable f in p(f * 0.95) } }) } catch let c as SpectroCancelled { throw c } diff --git a/Sources/SpectroEDX/Quant/PooledQuantification.swift b/Sources/SpectroEDX/Quant/PooledQuantification.swift index 97bebbf..519ba34 100644 --- a/Sources/SpectroEDX/Quant/PooledQuantification.swift +++ b/Sources/SpectroEDX/Quant/PooledQuantification.swift @@ -291,6 +291,9 @@ public nonisolated enum PooledQuantifier { fitSettings.resolutionMnKaEV = resolution let fitRange = range(axis) var deadLayerMicrons: Double? + // The fitted dead layer's warning, finished after the sum-peak decision (whether the final fit includes the sum/escape + // model is known only then), and its place in `warnings`. + var deadLayerNote: (text: String, at: Int)? if policy.fit.continuumDeadLayer != nil, deadLayerTable == nil { warnings.append("the Si dead-layer term applies to the continuum background only; not used with the polynomial background") } @@ -309,9 +312,8 @@ public nonisolated enum PooledQuantifier { } deadLayerMicrons = r.microns fitSettings = DeadLayerFit.settings(fitSettings, base: baseEfficiency, microns: r.microns, table: table) - warnings.append(String(format: "Si dead layer fitted: %.4f \u{00B5}m; the area sigmas are conditional on it", r.microns) - + (input.sumEscape.map { !$0.isOff } == true ? "; fitted without the sum/escape model, which the final fit includes" : "") - + (locked ? "" : "; the axis refinement ran without it")) + deadLayerNote = (String(format: "Si dead layer fitted: %.4f \u{00B5}m; the area sigmas are conditional on it", r.microns), + warnings.count) if r.atUpperBound { warnings.append(String(format: "the fitted Si dead layer sits at its upper bound (%.3g \u{00B5}m): the bound, not the data, set it", d.maximumMicrons)) } @@ -393,6 +395,11 @@ public nonisolated enum PooledQuantifier { let why = (error as? LocalizedError)?.errorDescription ?? "\(error)" throw QuantificationRefusal("The fit could not run: \(why).") } + if let note = deadLayerNote { + // `applied` is the sum/escape model the final fit ran with, from the caller or from `policy.fit.sumPeaks`. + warnings.insert(note.text + (applied != nil ? "; fitted without the sum/escape model, which the final fit includes" : "") + + (locked ? "" : "; the axis refinement ran without it"), at: note.at) + } // The sum and escape peaks the kept model explains, on the full axis (0 outside the fitted channels). var sumEscapeCounts: [Double]? if let r = keptSumEscape { @@ -543,6 +550,20 @@ public nonisolated enum PooledQuantifier { q.appliedSumEscape ?? SumEscapeModel() } + /// `refit` (the reported fit's input with one thing changed) set to repeat `q`'s data-driven fit choices instead of taking them + /// again on its own data: the sum-peak decision (`sameSumEscape(as:)`) and, when `q` fitted the Si dead layer, its thickness, + /// typed. A typed thickness is left as it is. When `q` ran on an unlocked axis without a refinement result (the refinement + /// failed and the file's axis was used), the refit is locked to the file's axis too, so it does not refine one of its own. + package static func pinningReportedChoices(_ refit: PooledQuantificationInput, as q: PooledQuantification) -> PooledQuantificationInput { + var out = refit + out.sumEscape = sameSumEscape(as: q) + if !q.axisLocked, q.refinement == nil { out.method.lockEnergyAxis = true } + if let d = out.policy.fit.continuumDeadLayer, d.microns == nil, let t = q.continuumDeadLayerMicrons { + out.policy.fit.continuumDeadLayer = .given(microns: t) + } + return out + } + /// What every shown sigma covers (WP3b F3 wording): the fit's counting statistics, not the choice of model. /// R7 (wp3e item 2): "\u{03C7}\u{00B2}\u{1D63} 54.8 (Pearson)" / "reduced deviance 1.02". Three significant figures at most; no /// threshold: the expectation of 1 is a property of the statistic, not a measured cliff. diff --git a/Sources/SpectroEDX/Quant/QuantificationPolicy.swift b/Sources/SpectroEDX/Quant/QuantificationPolicy.swift index 630e150..277dc47 100644 --- a/Sources/SpectroEDX/Quant/QuantificationPolicy.swift +++ b/Sources/SpectroEDX/Quant/QuantificationPolicy.swift @@ -84,7 +84,8 @@ public nonisolated struct FitPolicy: Sendable, Equatable, Codable { /// The detector Si dead-layer term of the continuum, typed or fitted (`ContinuumDeadLayer`); it multiplies /// `continuumEfficiency` when both are set. Default nil: off. It is not neutral: on the planted pool without a dead layer /// (`eds-absorption-truth.json`) a fitted thickness reads 0.21 um and moves Si K-alpha by -2.3 sigma (the step takes up part of - /// the sample-side continuum misfit above the Al K edge). A typed 0 is the default fit exactly. + /// the sample-side continuum misfit above the Al K edge); measured with sum peaks off, on the fit without the sum/escape model. + /// A typed 0 is the default fit exactly. public var continuumDeadLayer: ContinuumDeadLayer? /// Fit a Si-escape copy of every line group above the Si K edge with an area of its own. Default true; false is the reference's. public var escapePeaks: Bool @@ -142,8 +143,9 @@ public nonisolated struct FitPolicy: Sendable, Equatable, Codable { continuumDeadLayer, lineWeights } - /// The synthesized encoding in a fixed key order. Every field with a value is written, `continuumEdgeStep` and `sumPeaks` - /// included; a stored policy without them reads the form it was run with (see `init(from:)`). + /// Every non-optional field is written, `sumPeaks`, `sumPeakSignificance`, `continuumEdgeStep` and `lineWeights` included, + /// default or not. An optional that is nil (`continuumEfficiency`, `lineTail`, `continuumDeadLayer`) is left out, so the + /// default policy has no `continuumDeadLayer` key. Key order is not fixed here; `QuantificationPolicy.canonicalJSON` sorts it. public func encode(to encoder: any Encoder) throws { var c = encoder.container(keyedBy: CodingKeys.self) try c.encode(fitFromKeV, forKey: .fitFromKeV) @@ -165,9 +167,10 @@ public nonisolated struct FitPolicy: Sendable, Equatable, Codable { try c.encode(lineWeights, forKey: .lineWeights) } - /// A policy stored before sum peaks were the default has no `sumPeaks`: it reads `.off`, the fit it was stored with (and gets the - /// non-default footer line). One stored before `continuumEdgeStep` existed reads `.sharp`, the form it was run with; one without `lineWeights` reads `.exspy`. Every other - /// field is required, as before. + /// A field added after policies were first stored reads, when its key is missing, the value that policy was run with: `sumPeaks` + /// `.off` (the fit before sum peaks were the default; it gets the non-default footer line), `sumPeakSignificance` 3, + /// `continuumEdgeStep` `.sharp`, `lineWeights` `.exspy`. A missing optional (`continuumEfficiency`, `lineTail`, + /// `continuumDeadLayer`) reads nil. Every other field is required. public init(from decoder: any Decoder) throws { let c = try decoder.container(keyedBy: CodingKeys.self) self.init(fitFromKeV: try c.decode(Double.self, forKey: .fitFromKeV), diff --git a/Sources/SpectroEDX/Quant/UnlistedLineCheck.swift b/Sources/SpectroEDX/Quant/UnlistedLineCheck.swift index ab8b4e0..9a7ad7e 100644 --- a/Sources/SpectroEDX/Quant/UnlistedLineCheck.swift +++ b/Sources/SpectroEDX/Quant/UnlistedLineCheck.swift @@ -280,7 +280,7 @@ public nonisolated enum UnlistedLineChecker { refit.method.elements.removeAll { e in elements.contains { $0.element == e.symbol } } refit.method.elements += elements.map { .init(symbol: $0.element, role: .fitOnly, isManual: false) } refit.refinement = q.refinement - refit.sumEscape = PooledQuantifier.sameSumEscape(as: q) // the reported fit's sum-peak decision, not a new one + refit = PooledQuantifier.pinningReportedChoices(refit, as: q) // the reported fit's sum-peak decision and dead layer, not new ones let after: PooledQuantification do { after = try PooledQuantifier.run(refit, tables: nil, cancelled: cancelled, progress: progress.map { p in { @Sendable f in p(f * 0.95) } }) } catch let c as SpectroCancelled { throw c } diff --git a/Tests/SpectroEDXTests/ContinuumDeadLayerTests.swift b/Tests/SpectroEDXTests/ContinuumDeadLayerTests.swift index 236e288..17ebb17 100644 --- a/Tests/SpectroEDXTests/ContinuumDeadLayerTests.swift +++ b/Tests/SpectroEDXTests/ContinuumDeadLayerTests.swift @@ -219,6 +219,55 @@ struct ContinuumDeadLayerTests { #expect(both.fit.backgroundLabel.contains("EPQ") && both.fit.backgroundLabel.contains("Si dead layer 0.3000")) } + /// The fitted thickness is fitted without the sum/escape model; the warning says so whenever the final fit includes it, also + /// when that model comes from `FitPolicy.sumPeaks` (no `sumEscape` passed), and not when the final fit has none. Red before: + /// the clause was read from `input.sumEscape` alone. + @Test func testTheFittedDeadLayerWarningNamesTheSumEscapeModelWhenThePolicyKeepsIt() throws { + let clause = "fitted without the sum/escape model" + func note(_ q: PooledQuantification) throws -> String { try #require(q.warnings.first { $0.hasPrefix("Si dead layer fitted:") }) } + let kept = try PooledQuantifier.run(Self.input(policy: QuantificationPolicy(fit: FitPolicy(sumPeaks: .always, continuumDeadLayer: .fitted()))), tables: nil) + #expect(kept.appliedSumEscape != nil) + let keptNote = try note(kept) + #expect(keptNote.contains(clause), Comment(rawValue: keptNote)) + let off = try PooledQuantifier.run(Self.input(policy: QuantificationPolicy(fit: FitPolicy(sumPeaks: .off, continuumDeadLayer: .fitted()))), tables: nil) + #expect(off.appliedSumEscape == nil) + let offNote = try note(off) + #expect(!offNote.contains(clause), Comment(rawValue: offNote)) + var passed = Self.input(policy: QuantificationPolicy(fit: FitPolicy(sumPeaks: .off, continuumDeadLayer: .fitted()))) + passed.sumEscape = SumEscapeModel(sumPeaks: true) + let passedNote = try note(PooledQuantifier.run(passed, tables: nil)) + #expect(passedNote.contains(clause), Comment(rawValue: passedNote)) + } + + /// The range-sensitivity refit repeats the reported fit's dead layer: its nets are those of the alternative range with the + /// reported thickness typed, not with a thickness fitted again on the longer range (which differs, so the check can tell). + /// The default range is cut to 10 keV so that the pool's 20 keV axis runs past it. Red before: the refit fitted its own. + @Test func testTheRangeSensitivityRefitKeepsTheReportedDeadLayer() throws { + let fit = FitPolicy(defaultUpperLimitKeV: 10, sumPeaks: .off, continuumDeadLayer: .fitted()) + let input = Self.input(policy: QuantificationPolicy(fit: fit)) + let q = try PooledQuantifier.run(input, tables: nil) + let t = try #require(q.continuumDeadLayerMicrons) + let to = try #require(FitRangeSensitivityCheck.alternative(for: q)) + let s = try #require(FitRangeSensitivityCheck.run(input: input, quantification: q)) + #expect(s.failure == nil && s.alternativeToKeV == to && !s.moves.isEmpty) + + var pinned = input + pinned.method.fitToKeV = to + pinned.refinement = q.refinement + pinned.sumEscape = SumEscapeModel() + pinned.policy.fit.continuumDeadLayer = .given(microns: t) + let expected = try PooledQuantifier.run(pinned, tables: nil) + var free = pinned + free.policy.fit.continuumDeadLayer = .fitted() + let refitted = try #require(try PooledQuantifier.run(free, tables: nil).continuumDeadLayerMicrons) + Self.note(String(format: "range check: reported %.4f um (to %g keV), fitted again on to %g keV %.4f um", t, q.fitRange.toKeV, to, refitted)) + #expect(abs(refitted - t) > 1e-3, "the longer range must fit another thickness for this test to discriminate") + for m in s.moves { + let row = try #require(expected.rows.first { $0.element == m.element }) + #expect(m.after == row.net, "\(m.element): refit \(m.after), with the reported \(t) um \(row.net)") + } + } + @Test func testPolicyRefusesANegativeOrEmptyDeadLayer() { #expect(FitPolicy(continuumDeadLayer: .given(microns: -0.1)).problem != nil) #expect(FitPolicy(continuumDeadLayer: .fitted(maximumMicrons: 0)).problem != nil) diff --git a/Tests/SpectroEDXTests/QuantificationPolicyTests.swift b/Tests/SpectroEDXTests/QuantificationPolicyTests.swift index a22dff0..87366fa 100644 --- a/Tests/SpectroEDXTests/QuantificationPolicyTests.swift +++ b/Tests/SpectroEDXTests/QuantificationPolicyTests.swift @@ -114,6 +114,55 @@ struct QuantificationPolicyTests { #expect(throws: QuantificationRefusal.self) { try Self.run(c, policy: Self.fit(FitPolicy(continuumOrders: [9]))) } } + /// Every FitPolicy field at a non-default value survives `canonicalJSON`: it decodes to an equal policy and re-encodes to the + /// same string. The default writes no `continuumDeadLayer` key; a stored policy without it reads nil, one without `sumPeaks` + /// reads `.off`. + @Test func testEveryNonDefaultFitPolicyFieldRoundTripsThroughCanonicalJSON() throws { + var irls = IRLSSettings(); irls.maxIterations = 50 + var refinement = AxisRefinementSettings(); refinement.refineGain = false + let f = FitPolicy(fitFromKeV: 0.3, defaultUpperLimitKeV: 15, continuumOrders: [7, 4], continuumEdges: [1.5], + continuumNonNegative: false, continuumEfficiency: DetectorLayers(aluminiumNm: 20), escapePeaks: false, + maximumPolynomialOrder: 6, irls: irls, escapeRatioWarning: 0.1, axisRefinement: refinement, + lineTail: LineTail(fraction: 0.02, slopeKeV: 0.1, source: "typed"), sumPeaks: .always, + sumPeakSignificance: 5, continuumEdgeStep: .sharp, continuumDeadLayer: .given(microns: 0.3), + lineWeights: .epq) + let d = FitPolicy.default + #expect(f.fitFromKeV != d.fitFromKeV && f.defaultUpperLimitKeV != d.defaultUpperLimitKeV && f.continuumOrders != d.continuumOrders + && f.continuumEdges != d.continuumEdges && f.continuumNonNegative != d.continuumNonNegative + && f.continuumEfficiency != d.continuumEfficiency && f.escapePeaks != d.escapePeaks + && f.maximumPolynomialOrder != d.maximumPolynomialOrder && f.irls != d.irls && f.escapeRatioWarning != d.escapeRatioWarning + && f.axisRefinement != d.axisRefinement && f.lineTail != d.lineTail && f.sumPeaks != d.sumPeaks + && f.sumPeakSignificance != d.sumPeakSignificance && f.continuumEdgeStep != d.continuumEdgeStep + && f.continuumDeadLayer != d.continuumDeadLayer && f.lineWeights != d.lineWeights) + let p = QuantificationPolicy(fit: f) + let json = p.canonicalJSON + let back = try JSONDecoder().decode(QuantificationPolicy.self, from: Data(json.utf8)) + #expect(back == p) + #expect(back.canonicalJSON == json) + + // The default has no dead-layer key; a stored policy without it, or without `sumPeaks`, reads nil and `.off`. + func fitObject(_ s: String) throws -> [String: Any] { + let o = try #require(try JSONSerialization.jsonObject(with: Data(s.utf8)) as? [String: Any]) + return try #require(o["fit"] as? [String: Any]) + } + func decode(fit: [String: Any], like s: String) throws -> QuantificationPolicy { + var o = try #require(try JSONSerialization.jsonObject(with: Data(s.utf8)) as? [String: Any]) + o["fit"] = fit + return try JSONDecoder().decode(QuantificationPolicy.self, from: JSONSerialization.data(withJSONObject: o)) + } + let defaultJSON = QuantificationPolicy.default.canonicalJSON + #expect(try fitObject(defaultJSON)["continuumDeadLayer"] == nil) + #expect(!defaultJSON.contains("continuumDeadLayer")) + var withoutDeadLayer = try fitObject(json) + #expect(withoutDeadLayer.removeValue(forKey: "continuumDeadLayer") != nil) + let noDeadLayer = try decode(fit: withoutDeadLayer, like: json) + #expect(noDeadLayer.fit.continuumDeadLayer == nil) + #expect(noDeadLayer.fit.lineWeights == .epq && noDeadLayer.fit.sumPeaks == .always) // the rest as stored + var withoutSumPeaks = try fitObject(defaultJSON) + #expect(withoutSumPeaks.removeValue(forKey: "sumPeaks") != nil) + #expect(try decode(fit: withoutSumPeaks, like: defaultJSON).fit.sumPeaks == .off) + } + // MARK: F1 the fit start @Test func testFitStartMovesTheFittedChannelsAndTheRefusalBound() throws {