diff --git a/ShimmerBluetooth/ShimmerBluetooth.xcodeproj/project.pbxproj b/ShimmerBluetooth/ShimmerBluetooth.xcodeproj/project.pbxproj index cf19962..d0ee776 100644 --- a/ShimmerBluetooth/ShimmerBluetooth.xcodeproj/project.pbxproj +++ b/ShimmerBluetooth/ShimmerBluetooth.xcodeproj/project.pbxproj @@ -33,6 +33,8 @@ 3AD7C6032BD23238000E5455 /* SensorTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3AD7C6022BD23238000E5455 /* SensorTest.swift */; }; 3AEB12F92BCE6F7E00B5F6F1 /* Shimmer3SpeedTestProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3AEB12F82BCE6F7E00B5F6F1 /* Shimmer3SpeedTestProtocol.swift */; }; 3AED20692B0AF79E0066A0F8 /* TimeSensor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3AED20682B0AF79E0066A0F8 /* TimeSensor.swift */; }; + DE10230B2C0100000000001B /* TimestampUnwrap.swift in Sources */ = {isa = PBXBuildFile; fileRef = DE10230A2C0100000000001A /* TimestampUnwrap.swift */; }; + DE10230D2C0100000000001D /* TimestampUnwrapVectorsTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = DE10230C2C0100000000001C /* TimestampUnwrapVectorsTest.swift */; }; 3AED206B2B0AFCD10066A0F8 /* ShimmerUtilities.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3AED206A2B0AFCD10066A0F8 /* ShimmerUtilities.swift */; }; 3AED206D2B0B10960066A0F8 /* ShimmerUtilitiesTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3AED206C2B0B10960066A0F8 /* ShimmerUtilitiesTest.swift */; }; 3AED20712B0C4BA60066A0F8 /* MagSensor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3AED20702B0C4BA60066A0F8 /* MagSensor.swift */; }; @@ -83,6 +85,8 @@ 3AD7C6022BD23238000E5455 /* SensorTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SensorTest.swift; sourceTree = ""; }; 3AEB12F82BCE6F7E00B5F6F1 /* Shimmer3SpeedTestProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Shimmer3SpeedTestProtocol.swift; sourceTree = ""; }; 3AED20682B0AF79E0066A0F8 /* TimeSensor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TimeSensor.swift; sourceTree = ""; }; + DE10230A2C0100000000001A /* TimestampUnwrap.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TimestampUnwrap.swift; sourceTree = ""; }; + DE10230C2C0100000000001C /* TimestampUnwrapVectorsTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TimestampUnwrapVectorsTest.swift; sourceTree = ""; }; 3AED206A2B0AFCD10066A0F8 /* ShimmerUtilities.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShimmerUtilities.swift; sourceTree = ""; }; 3AED206C2B0B10960066A0F8 /* ShimmerUtilitiesTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShimmerUtilitiesTest.swift; sourceTree = ""; }; 3AED20702B0C4BA60066A0F8 /* MagSensor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MagSensor.swift; sourceTree = ""; }; @@ -163,6 +167,7 @@ 3AED20702B0C4BA60066A0F8 /* MagSensor.swift */, 3AB320E92B07092C003D94F8 /* WRAccelSensor.swift */, 3AED20682B0AF79E0066A0F8 /* TimeSensor.swift */, + DE10230A2C0100000000001A /* TimestampUnwrap.swift */, 3AB320EB2B0709DA003D94F8 /* ObjectCluster.swift */, 3AED206A2B0AFCD10066A0F8 /* ShimmerUtilities.swift */, 92B106792B7DAFBC00AB9952 /* EXGSensor.swift */, @@ -178,6 +183,7 @@ 3A780C942AE12BEB00EAF050 /* ShimmerBluetoothTests.swift */, 3AED206C2B0B10960066A0F8 /* ShimmerUtilitiesTest.swift */, 3AD7C6022BD23238000E5455 /* SensorTest.swift */, + DE10230C2C0100000000001C /* TimestampUnwrapVectorsTest.swift */, ); path = ShimmerBluetoothTests; sourceTree = ""; @@ -314,6 +320,7 @@ 3A77128D2AE177A4006213A8 /* ShimmerDevice.swift in Sources */, 3AA359152AF37943008AD334 /* ConfigByteLayoutShimmer3.swift in Sources */, 3AED20692B0AF79E0066A0F8 /* TimeSensor.swift in Sources */, + DE10230B2C0100000000001B /* TimestampUnwrap.swift in Sources */, 9299E50E2B85054E001EEFE0 /* PressureTempSensor.swift in Sources */, 3A0B4B652B0DDE0000786295 /* GSRSensor.swift in Sources */, 3AB320EC2B0709DA003D94F8 /* ObjectCluster.swift in Sources */, @@ -339,6 +346,7 @@ 3AED206D2B0B10960066A0F8 /* ShimmerUtilitiesTest.swift in Sources */, 3A780C952AE12BEB00EAF050 /* ShimmerBluetoothTests.swift in Sources */, 3AD7C6032BD23238000E5455 /* SensorTest.swift in Sources */, + DE10230D2C0100000000001D /* TimestampUnwrapVectorsTest.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/ShimmerBluetooth/ShimmerBluetooth/ObjectCluster.swift b/ShimmerBluetooth/ShimmerBluetooth/ObjectCluster.swift index 8a5d342..4c13ca8 100644 --- a/ShimmerBluetooth/ShimmerBluetooth/ObjectCluster.swift +++ b/ShimmerBluetooth/ShimmerBluetooth/ObjectCluster.swift @@ -13,6 +13,11 @@ public class ObjectCluster { public var SignalData : [Double] = [] let Seperator = "_" public var PacketReceptionRate = -1 + /// False when the packet carried no usable timestamp - see + /// `TimestampUnwrap`. The sensor values on this cluster are real; only its + /// time is missing, and the Time Stamp signals repeat the previous + /// packet's. Drop the cluster if you need a true time axis. + public var timestampValid = true public func addData(sensorName:String,formatName:String,unitName:String,value:Double){ let newName = [sensorName,Seperator,formatName,Seperator,unitName].joined() SignalNames.append(newName) diff --git a/ShimmerBluetooth/ShimmerBluetooth/Shimmer3Protocol.swift b/ShimmerBluetooth/ShimmerBluetooth/Shimmer3Protocol.swift index 0e5481e..4d0a9d1 100644 --- a/ShimmerBluetooth/ShimmerBluetooth/Shimmer3Protocol.swift +++ b/ShimmerBluetooth/ShimmerBluetooth/Shimmer3Protocol.swift @@ -1074,6 +1074,16 @@ public class Shimmer3Protocol : NSObject, ShimmerProtocol { packetSize = TimeStampPacketByteSize // Time stamp timeSensor.packetIndexTimeStamp = 0 timeSensor.sensorEnabled = true + /* Both callers set CurrentSamplingRate from the inquiry response just + above, and the rate is what sizes the reorder window: eight sample + periods is what separates a pair of packets delivered out of order + from a dropout that happens to span the counter's roll-over point. + A rate that arrives as a division by zero is +Infinity here, and + reorderWindowTicks answers 0 for it - the branch off is the only + safe reading of an unknown rate. */ + timeSensor.reorderWindowTicks = TimestampUnwrap.reorderWindowTicks( + samplingRateHz: CurrentSamplingRate, + maxTicks: timeSensor.TimeStampPacketRawMaxTicks) //} var enabledSensors = Int(0) diff --git a/ShimmerBluetooth/ShimmerBluetooth/TimeSensor.swift b/ShimmerBluetooth/ShimmerBluetooth/TimeSensor.swift index 2d1005c..23c1169 100644 --- a/ShimmerBluetooth/ShimmerBluetooth/TimeSensor.swift +++ b/ShimmerBluetooth/ShimmerBluetooth/TimeSensor.swift @@ -14,24 +14,70 @@ public class TimeSensor : Sensor , SensorProcessing{ public func processData(sensorPacket: [UInt8], objectCluster: ObjectCluster) -> ObjectCluster { let desiredRange = Array(sensorPacket[packetIndexTimeStamp.. Double { - if (LastReceivedTimeStamp > (timeStamp + (TimeStampPacketRawMaxValue * CurrentTimeStampCycle))) - { - CurrentTimeStampCycle = CurrentTimeStampCycle + 1; - } - LastReceivedTimeStamp = (timeStamp + (TimeStampPacketRawMaxValue * CurrentTimeStampCycle)); + let result = TimestampUnwrap.unwrap(rawTicks: timeStamp, + lastUnwrapped: LastReceivedTimeStamp, + cycle: CurrentTimeStampCycle, + maxTicks: TimeStampPacketRawMaxTicks, + reorderWindowTicks: reorderWindowTicks, + hasPreviousSample: hasPreviousTimeStamp) + hasPreviousTimeStamp = true + lastRecordRejected = result.rejected + LastReceivedTimeStamp = result.unwrappedTicks + CurrentTimeStampCycle = result.cycle let clockConstant:Double = 32768; let calibratedTimeStamp = LastReceivedTimeStamp / clockConstant * 1000; // to convert into mS diff --git a/ShimmerBluetooth/ShimmerBluetooth/TimestampUnwrap.swift b/ShimmerBluetooth/ShimmerBluetooth/TimestampUnwrap.swift new file mode 100644 index 0000000..c05e123 --- /dev/null +++ b/ShimmerBluetooth/ShimmerBluetooth/TimestampUnwrap.swift @@ -0,0 +1,221 @@ +// +// TimestampUnwrap.swift +// ShimmerBluetooth +// +// Created by Shimmer Engineering on 17/09/2026. +// + +import Foundation + +/// Turning a device's wrapping sample counter into a monotonic clock. +/// +/// The counter runs at 32768 Hz in 24 bits, so it returns to zero every 512 +/// seconds exactly. Undoing that is a matter of counting the roll-overs, and +/// the obvious rule — "the value went down, so it wrapped" — is wrong three +/// different ways: +/// +/// - A **reordered** packet steps backwards without the counter having wrapped. +/// - A **duplicated** packet does the same with a step of zero. +/// - An **unstamped** record carries `0x000000`. Firmware stamps a packet when +/// the sample tick starts it and does not publish a packet it never stamped, +/// so a timestamp field of exactly zero marks an invalid record rather than +/// the counter reaching its origin. LogAndStream v1.00.x–v1.01.003 could +/// produce one under SD write back-pressure. +/// +/// Each of those adds a whole modulo — 512 seconds — to every later sample, for +/// the rest of the session. A recording of 9 minutes 30 seconds containing four +/// unstamped records was reported as 43 minutes 38. +/// +/// This is one of five implementations of the same rule, and the only one whose +/// correctness is not a matter of opinion: the vectors in +/// `TimestampUnwrapVectorsTest` are generated from a reference implementation in +/// the firmware repository, where the rule is specified, and the Java, C#, +/// Python and TypeScript APIs run the same set. They drifted apart once already +/// — the same defect, in all five, for years — because each was reviewed on its +/// own against prose. Change the rule here and the vectors will say so. +/// +/// Mirrors `TimestampUnwrap.cs` and `TimestampUnwrap.java` method for method, +/// with one deliberate difference: both of those keep an overload that infers +/// "no previous sample" from the state being `(0, 0)`, for callers written +/// before that turned out to be ambiguous. This file is new, so it has no such +/// callers and asks outright instead. +public enum TimestampUnwrap { + + /// The 3-byte counter's range. The only width this API's packet parser + /// reads (`Shimmer3Protocol.TimeStampPacketByteSize`), but the rule is + /// written against a modulo so that the shared vectors can exercise the + /// 2-byte counter older firmware used. + public static let ticksMax3Byte = 1 << 24 + + /// How close to the top of the range the previous sample must have been for + /// a drop to exactly zero to be believed as a roll-over. + /// + /// One second. A genuine wrap onto zero means the counter advanced to its + /// very last tick, so its predecessor is within a sample or two of the + /// maximum; a second is a generous allowance for a gap in the data, and + /// orders of magnitude away from the mid-range predecessors the unstamped + /// records have. + public static let wrapWindowTicks = 32768 + + /// How many sample periods behind its predecessor a value may be and still + /// be read as a reordered packet. + /// + /// A reorder swaps packets that are adjacent in time, so it spans a handful + /// of sample periods; a dropout spans whatever the link lost. Eight periods + /// sits orders of magnitude clear of both at any rate the hardware offers. + public static let reorderPeriods = 8 + + /// The real-time clock the packet counter runs on. + /// + /// **Not** a TCXO sampling clock (312500 Hz, or 255765.625 Hz), which some + /// firmware uses to derive the rate. Sizing the window in that domain makes + /// it 9.5x too wide. + public static let rtcTicksPerSecond = 32768.0 + + /// The largest fraction of the counter's range a window may occupy. + /// + /// At 1 Hz on the 2-byte counter eight sample periods is four whole + /// modulos, and a window at or above the modulo leaves no backward step + /// large enough to be a roll-over — the unwrap would stop counting them + /// altogether. + public static let maxWindowDivisor = 8 + + /// One sample's place on the timeline. + public struct Result { + /// The counter with its roll-overs added back. **Not monotonic**: a + /// reordered packet reports the position it actually holds, which is + /// behind the sample before it. + public let unwrappedTicks: Double + /// How many whole modulos `unwrappedTicks` is above the counter's + /// origin. Carried as state by the caller; it may dip by one for a + /// packet arriving late from before a boundary. + public let cycle: Double + /// True when the record carried no usable timestamp. The timeline is + /// held where it was, so `unwrappedTicks` and `cycle` repeat the + /// previous sample's and say nothing about when this one was taken. + /// The sensor values are real; only the time is missing. + public let rejected: Bool + + public init(unwrappedTicks: Double, cycle: Double, rejected: Bool) { + self.unwrappedTicks = unwrappedTicks + self.cycle = cycle + self.rejected = rejected + } + } + + /// The reorder window for a stream at a known sampling rate, in ticks. + /// + /// Sized in **sample periods**, not as a fraction of the counter's range. + /// The two are easy to confuse and behave very differently: a reorder swaps + /// adjacent packets, whereas a dropout that happens to span the wrap point + /// is most of a modulo. Sizing the window by the modulo puts the boundary + /// between them in the middle of ordinary dropout territory — at 2^16 every + /// gap between 1.75 s and 2.0 s reads as a reorder and the roll-over is + /// silently lost. Eight sample periods shrinks that misread band to about + /// 16 ms. + /// + /// `0` — the branch disabled — when the rate is not a usable number. Never + /// guess: an unknown rate must not become an infinite window, which would + /// read every backward step as a reorder and lose every roll-over. That is + /// a worse failure than no reorder detection at all, because the recording + /// still looks plausible. + /// + /// - Parameters: + /// - samplingRateHz: Samples per second, in the 32768 Hz tick domain. + /// - maxTicks: The counter's range. + public static func reorderWindowTicks(samplingRateHz: Double, maxTicks: Int) -> Double { + if samplingRateHz.isNaN || samplingRateHz.isInfinite || samplingRateHz <= 0.0 { + return 0.0 + } + let window = Double(reorderPeriods) * rtcTicksPerSecond / samplingRateHz + return min(window, Double(maxTicks) / Double(maxWindowDivisor)) + } + + /// The rule, with reorder detection disabled. + /// + /// `hasPreviousSample` is asked for here too. Defaulting it would be a quiet + /// way to get the first sample of a stream wrong, and there is no caller + /// older than this file to keep compatible - unlike the C# and Java copies, + /// which keep an overload that infers it from `(0, 0)` for exactly that + /// reason. + public static func unwrap(rawTicks: Double, lastUnwrapped: Double, cycle: Double, + maxTicks: Int, hasPreviousSample: Bool) -> Result { + return unwrap(rawTicks: rawTicks, lastUnwrapped: lastUnwrapped, cycle: cycle, + maxTicks: maxTicks, reorderWindowTicks: 0.0, + hasPreviousSample: hasPreviousSample) + } + + /// Place one sample on the timeline. + /// + /// Every decision is made on the **modular forward distance** from the + /// previous sample — never by comparing candidate unwrapped values, which + /// looks equivalent and is not. A packet arriving late from just before a + /// boundary has a candidate *above* its predecessor, so a comparison + /// accepts it as forward motion of nearly a whole modulo and then reads the + /// next real sample as a second roll-over: `[2^24 - 10, 5, 2^24 - 10, 70]` + /// lands at 33554502, two modulos out, from one out-of-order packet. + /// + /// Forward motion is the **default**, which is what keeps a roll-over + /// preceded by a long dropout classified as a roll-over: however much was + /// lost, the counter still wrapped. A rule that defaults the other way — + /// "a backward step is corrupt unless it clears some threshold" — fails + /// exactly there. + /// + /// - Parameters: + /// - rawTicks: The counter value out of the packet. + /// - lastUnwrapped: The previous accepted sample's unwrapped value. + /// - cycle: The previous accepted sample's cycle. + /// - maxTicks: The counter's range. + /// - reorderWindowTicks: From `reorderWindowTicks(samplingRateHz:maxTicks:)`. + /// - hasPreviousSample: False only before the first sample of a stream. + /// Asked outright rather than inferred from `(0, 0)`, which is the reset + /// state *and* a state this rule can reach: a reorder that lands exactly + /// on the counter's origin leaves both at zero in the middle of a + /// stream, after which the next packet is read as a first sample and + /// passed through - so one arriving from just before the origin is + /// placed a whole modulo late rather than a few ticks behind. The + /// conformance vector + /// `reorder-onto-origin-then-earlier-packet-24bit` is that sequence. + /// Hosts that keep the previous raw value instead of a cycle count, as + /// the web SDK and pyshimmer do, never had the ambiguity. + public static func unwrap(rawTicks: Double, lastUnwrapped: Double, cycle: Double, + maxTicks: Int, reorderWindowTicks: Double, + hasPreviousSample: Bool) -> Result { + if !hasPreviousSample { + // No predecessor to measure against. Taking the reset state as a real + // sample at zero would let a first raw value near the top of the range + // read as a packet reordered across a boundary, placing a whole + // recording one modulo early. + return Result(unwrappedTicks: rawTicks, cycle: 0.0, rejected: false) + } + + let modulo = Double(maxTicks) + let lastRaw = lastUnwrapped - (modulo * cycle) + var forward = rawTicks - lastRaw + if forward < 0 { forward += modulo } + let backwards = modulo - forward + + let candidate: Double + if forward == 0.0 { + // A duplicate: hold the timeline where it is. + candidate = lastUnwrapped + } else if backwards <= reorderWindowTicks { + // Reordered, on either side of a boundary. Placed where it was + // actually taken, which is below its predecessor — honest rather + // than monotonic. + candidate = lastUnwrapped - backwards + } else if maxTicks == ticksMax3Byte && rawTicks == 0.0 + && lastRaw < modulo - Double(wrapWindowTicks) { + // A record the firmware never stamped. Nothing about it moves the + // state, so the next real sample reads as the ordinary step forward + // it is rather than as a second roll-over. + return Result(unwrappedTicks: lastUnwrapped, cycle: cycle, rejected: true) + } else { + candidate = lastUnwrapped + forward + } + + return Result(unwrappedTicks: candidate, + cycle: (candidate / modulo).rounded(.down), + rejected: false) + } +} diff --git a/ShimmerBluetooth/ShimmerBluetoothTests/TimestampUnwrapVectorsTest.swift b/ShimmerBluetooth/ShimmerBluetoothTests/TimestampUnwrapVectorsTest.swift new file mode 100644 index 0000000..6f3b6d1 --- /dev/null +++ b/ShimmerBluetooth/ShimmerBluetoothTests/TimestampUnwrapVectorsTest.swift @@ -0,0 +1,586 @@ +// +// TimestampUnwrapVectorsTest.swift +// ShimmerBluetoothTests +// +// Created by Shimmer Engineering on 17/09/2026. +// + +import XCTest +@testable import ShimmerBluetooth + +/// The timestamp-unwrap rule, run against the vectors every Shimmer host API is +/// checked against. +/// +/// The set is specified and generated in the firmware repository - +/// `log-and-stream-common`, `Test/conformance/timestamp_unwrap.json`, beside the +/// prose it encodes and a reference implementation that regenerates and +/// re-checks it in CI. The Java, C#, Python and TypeScript APIs load that file +/// directly; a classic Xcode project would need four hand-edited entries in +/// project.pbxproj to carry a resource nothing compiles, so this target gets the +/// same vectors as generated source instead: +/// +/// python Test/host/crosscheck_timestamp_unwrap.py --emit swift +/// +/// Why go to that trouble for what looks like arithmetic: five implementations +/// of one wire format drifted apart once already, and the same unwrap defect sat +/// in all five for years - each reviewed on its own, against prose, by people +/// who had no way to run the others. A shared set turns "these should agree" +/// into something a suite fails on. + +/// One vector: a raw counter sequence and what every host must make of it. +struct UnwrapVector { + let id: String + let modulo: Int + let reorderWindowTicks: Double + let raw: [Double] + let expectedUnwrapped: [Double] + let expectedRejected: [Bool] + let expectedFinalCycle: Int +} + +/// One window derivation: a sampling rate and the window it must produce. +struct WindowDerivationCase { + let samplingRateHz: Double? + let timestampBits: Int + let expectedReorderWindowTicks: Double + let tolerance: Double +} + +// Generated by Test/host/crosscheck_timestamp_unwrap.py --emit swift +// Source: Test/conformance/timestamp_unwrap.json revision 1 +// Do not hand-edit; regenerate when the vector file changes. + +let sharedVectorRevision = 1 +let sharedTicksPerSecond = 32768 +let sharedInvalidZeroWindowTicks = 32768 +let sharedReorderPeriods = 8 +let sharedMaxWindowDivisor = 8 + +let sharedUnwrapVectors: [UnwrapVector] = [ + UnwrapVector( + id: "monotonic-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [1000, 1065, 1130], + expectedUnwrapped: [1000, 1065, 1130], + expectedRejected: [false, false, false], + expectedFinalCycle: 0), + UnwrapVector( + id: "wrap-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [16777200, 16], + expectedUnwrapped: [16777200, 16777232], + expectedRejected: [false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "wrap-lands-on-zero-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [16777116, 0], + expectedUnwrapped: [16777116, 16777216], + expectedRejected: [false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "invalid-zero-signature-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [7406116, 7406506, 0, 7406571], + expectedUnwrapped: [7406116, 7406506, 7406506, 7406571], + expectedRejected: [false, false, true, false], + expectedFinalCycle: 0), + UnwrapVector( + id: "invalid-zero-no-cascade-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [7406506, 0, 7406571, 7406636], + expectedUnwrapped: [7406506, 7406506, 7406571, 7406636], + expectedRejected: [false, true, false, false], + expectedFinalCycle: 0), + UnwrapVector( + id: "first-sample-zero-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [0, 65, 130], + expectedUnwrapped: [0, 65, 130], + expectedRejected: [false, false, false], + expectedFinalCycle: 0), + UnwrapVector( + id: "wrap-16bit", + modulo: 65536, + reorderWindowTicks: 5120.0, + raw: [65436, 28], + expectedUnwrapped: [65436, 65564], + expectedRejected: [false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "zero-on-16bit-is-a-wrap", + modulo: 65536, + reorderWindowTicks: 5120.0, + raw: [30000, 0], + expectedUnwrapped: [30000, 65536], + expectedRejected: [false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "backward-step-outside-window-is-a-wrap-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [7406506, 1], + expectedUnwrapped: [7406506, 16777217], + expectedRejected: [false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "duplicate-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [1000, 1065, 1065, 1130], + expectedUnwrapped: [1000, 1065, 1065, 1130], + expectedRejected: [false, false, false, false], + expectedFinalCycle: 0), + UnwrapVector( + id: "reorder-one-period-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [1000, 1130, 1065, 1195], + expectedUnwrapped: [1000, 1130, 1065, 1195], + expectedRejected: [false, false, false, false], + expectedFinalCycle: 0), + UnwrapVector( + id: "reorder-one-period-16bit", + modulo: 65536, + reorderWindowTicks: 5120.0, + raw: [40000, 41280, 40640, 41920], + expectedUnwrapped: [40000, 41280, 40640, 41920], + expectedRejected: [false, false, false, false], + expectedFinalCycle: 0), + UnwrapVector( + id: "reorder-across-wrap-boundary-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [16777206, 5, 16777206, 70], + expectedUnwrapped: [16777206, 16777221, 16777206, 16777286], + expectedRejected: [false, false, false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "wrap-after-heavy-loss-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [16000000, 100], + expectedUnwrapped: [16000000, 16777316], + expectedRejected: [false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "wrap-after-heavy-loss-16bit", + modulo: 65536, + reorderWindowTicks: 5120.0, + raw: [60000, 1000], + expectedUnwrapped: [60000, 66536], + expectedRejected: [false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "wrap-spanning-dropout-1p8s-16bit", + modulo: 65536, + reorderWindowTicks: 5120.0, + raw: [60000, 53446], + expectedUnwrapped: [60000, 118982], + expectedRejected: [false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "wrap-spanning-dropout-152s-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [16000000, 4222784], + expectedUnwrapped: [16000000, 21000000], + expectedRejected: [false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "rate-unknown-backward-step-is-a-wrap-24bit", + modulo: 16777216, + reorderWindowTicks: 0.0, + raw: [1000, 1130, 1065, 1195], + expectedUnwrapped: [1000, 1130, 16778281, 16778411], + expectedRejected: [false, false, false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "rate-unknown-zero-still-rejected-24bit", + modulo: 16777216, + reorderWindowTicks: 0.0, + raw: [7406506, 0, 7406571], + expectedUnwrapped: [7406506, 7406506, 7406571], + expectedRejected: [false, true, false], + expectedFinalCycle: 0), + UnwrapVector( + id: "zero-within-window-of-origin-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [300, 365, 0, 430], + expectedUnwrapped: [300, 365, 0, 430], + expectedRejected: [false, false, false, false], + expectedFinalCycle: 0), + UnwrapVector( + id: "zero-within-window-after-wrap-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [16777100, 100, 165, 0, 230], + expectedUnwrapped: [16777100, 16777316, 16777381, 16777216, 16777446], + expectedRejected: [false, false, false, false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "reorder-window-boundary-inclusive-24bit", + modulo: 16777216, + reorderWindowTicks: 512.0, + raw: [10512, 10000], + expectedUnwrapped: [10512, 10000], + expectedRejected: [false, false], + expectedFinalCycle: 0), + UnwrapVector( + id: "reorder-window-boundary-exclusive-24bit", + modulo: 16777216, + reorderWindowTicks: 512.0, + raw: [10513, 10000], + expectedUnwrapped: [10513, 16787216], + expectedRejected: [false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "low-rate-clamp-16bit", + modulo: 65536, + reorderWindowTicks: 8192.0, + raw: [60000, 1000], + expectedUnwrapped: [60000, 66536], + expectedRejected: [false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "high-rate-reorder-24bit", + modulo: 16777216, + reorderWindowTicks: 256.0, + raw: [5000, 5032, 5000, 5064], + expectedUnwrapped: [5000, 5032, 5000, 5064], + expectedRejected: [false, false, false, false], + expectedFinalCycle: 0), + UnwrapVector( + id: "reorder-beyond-eight-periods-is-a-wrap-24bit", + modulo: 16777216, + reorderWindowTicks: 256.0, + raw: [5000, 5288, 5000], + expectedUnwrapped: [5000, 5288, 16782216], + expectedRejected: [false, false, false], + expectedFinalCycle: 1), + UnwrapVector( + id: "reorder-onto-origin-then-earlier-packet-24bit", + modulo: 16777216, + reorderWindowTicks: 520.0, + raw: [520, 0, 16777200], + expectedUnwrapped: [520, 0, -16], + expectedRejected: [false, false, false], + expectedFinalCycle: -1), +] + +let sharedWindowDerivationCases: [WindowDerivationCase] = [ + WindowDerivationCase( + samplingRateHz: 504.12307692307695, + timestampBits: 24, + expectedReorderWindowTicks: 520.0, + tolerance: 0.0), + WindowDerivationCase( + samplingRateHz: 51.2, + timestampBits: 16, + expectedReorderWindowTicks: 5120.0, + tolerance: 0.0), + WindowDerivationCase( + samplingRateHz: 51.2, + timestampBits: 24, + expectedReorderWindowTicks: 5120.0, + tolerance: 0.0), + WindowDerivationCase( + samplingRateHz: 512.0, + timestampBits: 24, + expectedReorderWindowTicks: 512.0, + tolerance: 0.0), + WindowDerivationCase( + samplingRateHz: 1024.0, + timestampBits: 24, + expectedReorderWindowTicks: 256.0, + tolerance: 0.0), + WindowDerivationCase( + samplingRateHz: 1.0, + timestampBits: 16, + expectedReorderWindowTicks: 8192.0, + tolerance: 0.0), + WindowDerivationCase( + samplingRateHz: 1.0, + timestampBits: 24, + expectedReorderWindowTicks: 262144.0, + tolerance: 0.0), + WindowDerivationCase( + samplingRateHz: 0.0, + timestampBits: 24, + expectedReorderWindowTicks: 0.0, + tolerance: 0.0), + WindowDerivationCase( + samplingRateHz: nil, + timestampBits: 24, + expectedReorderWindowTicks: 0.0, + tolerance: 0.0), + WindowDerivationCase( + samplingRateHz: -5.0, + timestampBits: 24, + expectedReorderWindowTicks: 0.0, + tolerance: 0.0), + WindowDerivationCase( + samplingRateHz: 512.2950819672132, + timestampBits: 24, + expectedReorderWindowTicks: 511.705088, + tolerance: 0.0001), +] + +final class TimestampUnwrapVectorsTest: XCTestCase { + + /// A bumped revision means the rule moved. Read the upstream change before + /// touching anything here. + func testRevisionIsTheOneThisSuiteWasWrittenAgainst() { + XCTAssertEqual(sharedVectorRevision, 1) + XCTAssertEqual(sharedTicksPerSecond, Int(TimestampUnwrap.rtcTicksPerSecond)) + XCTAssertEqual(sharedInvalidZeroWindowTicks, TimestampUnwrap.wrapWindowTicks) + XCTAssertEqual(sharedReorderPeriods, TimestampUnwrap.reorderPeriods) + XCTAssertEqual(sharedMaxWindowDivisor, TimestampUnwrap.maxWindowDivisor) + } + + /// A vector that stops being run is a vector that stops protecting + /// anything, and a loop over whatever the array happens to hold would not + /// notice. Updating this list is the moment to ask what changed upstream. + func testVectorSetIsComplete() { + let expectedIds = [ + "monotonic-24bit", + "wrap-24bit", + "wrap-lands-on-zero-24bit", + "invalid-zero-signature-24bit", + "invalid-zero-no-cascade-24bit", + "first-sample-zero-24bit", + "wrap-16bit", + "zero-on-16bit-is-a-wrap", + "backward-step-outside-window-is-a-wrap-24bit", + "duplicate-24bit", + "reorder-one-period-24bit", + "reorder-one-period-16bit", + "reorder-across-wrap-boundary-24bit", + "wrap-after-heavy-loss-24bit", + "wrap-after-heavy-loss-16bit", + "wrap-spanning-dropout-1p8s-16bit", + "wrap-spanning-dropout-152s-24bit", + "rate-unknown-backward-step-is-a-wrap-24bit", + "rate-unknown-zero-still-rejected-24bit", + "zero-within-window-of-origin-24bit", + "zero-within-window-after-wrap-24bit", + "reorder-window-boundary-inclusive-24bit", + "reorder-window-boundary-exclusive-24bit", + "low-rate-clamp-16bit", + "high-rate-reorder-24bit", + "reorder-beyond-eight-periods-is-a-wrap-24bit", + "reorder-onto-origin-then-earlier-packet-24bit" + ] + XCTAssertEqual(sharedUnwrapVectors.map { $0.id }, expectedIds) + XCTAssertEqual(sharedWindowDerivationCases.count, 11) + } + + func testAllSharedVectorsAgree() { + for vector in sharedUnwrapVectors { + var lastUnwrapped = 0.0 + var cycle = 0.0 + var hasPrevious = false + + for (i, raw) in vector.raw.enumerated() { + let result = TimestampUnwrap.unwrap(rawTicks: raw, + lastUnwrapped: lastUnwrapped, + cycle: cycle, + maxTicks: vector.modulo, + reorderWindowTicks: vector.reorderWindowTicks, + hasPreviousSample: hasPrevious) + hasPrevious = true + XCTAssertEqual(result.unwrappedTicks, vector.expectedUnwrapped[i], accuracy: 0.0, + "\(vector.id): sample \(i)") + XCTAssertEqual(result.rejected, vector.expectedRejected[i], + "\(vector.id): sample \(i) rejected") + lastUnwrapped = result.unwrappedTicks + cycle = result.cycle + } + + XCTAssertEqual(cycle, Double(vector.expectedFinalCycle), accuracy: 0.0, + "\(vector.id): final cycle") + } + } + + func testWindowDerivationCases() { + for testCase in sharedWindowDerivationCases { + let modulo = 1 << testCase.timestampBits + // A rate the file spells as null is a rate the host does not know. + // Swift has no such Double, so it arrives here as a NaN - which is + // exactly the shape an unknown rate takes in a real host. + let rate = testCase.samplingRateHz ?? Double.nan + let got = TimestampUnwrap.reorderWindowTicks(samplingRateHz: rate, maxTicks: modulo) + XCTAssertEqual(got, testCase.expectedReorderWindowTicks, + accuracy: max(testCase.tolerance, 1e-9), + "window for rate \(String(describing: testCase.samplingRateHz)) " + + "at \(testCase.timestampBits) bits") + } + } + + /// The values the shared set cannot spell. + /// + /// An unknown rate arriving as a division result rather than as a literal + /// zero is the case that matters: 32768 / 0 is +Infinity in Swift, not an + /// error, and a host that lets it through gets a window wider than the + /// modulo. Every backward step then reads as a reorder, every roll-over is + /// lost, and nothing looks wrong until a recording comes out 512 s short. + func testWindowIsZeroForEveryShapeOfUnknownRate() { + let mod = TimestampUnwrap.ticksMax3Byte + XCTAssertEqual(TimestampUnwrap.reorderWindowTicks(samplingRateHz: .infinity, maxTicks: mod), 0.0) + XCTAssertEqual(TimestampUnwrap.reorderWindowTicks(samplingRateHz: .nan, maxTicks: mod), 0.0) + XCTAssertEqual(TimestampUnwrap.reorderWindowTicks(samplingRateHz: 0.0, maxTicks: mod), 0.0) + XCTAssertEqual(TimestampUnwrap.reorderWindowTicks(samplingRateHz: -5.0, maxTicks: mod), 0.0) + // The shape it actually arrives in: interpretInquiryResponseShimmer3 + // divides 32768 by the ADC value out of the packet. + XCTAssertEqual(TimestampUnwrap.reorderWindowTicks(samplingRateHz: 32768.0 / 0.0, maxTicks: mod), 0.0) + } + + // MARK: - TimeSensor, which is what actually runs the rule + + /// 51.2 Hz is a divider of 640 ticks, so eight sample periods is 5120. + private let windowAt51Point2Hz = TimestampUnwrap.reorderWindowTicks( + samplingRateHz: 51.2, maxTicks: TimestampUnwrap.ticksMax3Byte) + + private func milliseconds(_ ticks: Double) -> Double { + return ticks / 32768.0 * 1000.0 + } + + func testTimeSensorPlacesASwappedPairWhereItWasTaken() { + XCTAssertEqual(windowAt51Point2Hz, 5120.0) + + let sensor = TimeSensor() + sensor.reorderWindowTicks = windowAt51Point2Hz + + // Two adjacent packets delivered the wrong way round. Each is placed + // when it was taken, so the series dips - honestly - rather than + // gaining a whole modulo. + let wire = [10000.0, 11280.0, 10640.0, 11920.0] + let got = wire.map { sensor.calibrateTimeStamp(timeStamp: $0) } + for (i, ticks) in wire.enumerated() { + XCTAssertEqual(got[i], milliseconds(ticks), accuracy: 1e-9, "sample \(i)") + } + XCTAssertEqual(sensor.CurrentTimeStampCycle, 0.0) + XCTAssertFalse(sensor.lastRecordRejected) + } + + /// What the reorder window buys, stated as the difference it makes. + /// + /// With the branch off - which is what an unknown sampling rate has to mean + /// - the same swapped pair is read as a roll-over and the recording gains + /// 512 seconds from that point on. This is the old behaviour, kept as a + /// test so that the value of wiring the rate through is not theoretical. + func testTimeSensorWithoutAWindowChargesAModuloForTheSameSwap() { + let sensor = TimeSensor() + sensor.reorderWindowTicks = 0.0 + + _ = sensor.calibrateTimeStamp(timeStamp: 10000) + _ = sensor.calibrateTimeStamp(timeStamp: 11280) + let late = sensor.calibrateTimeStamp(timeStamp: 10640) + + XCTAssertEqual(late, milliseconds(10640 + Double(TimestampUnwrap.ticksMax3Byte)), + accuracy: 1e-9) + XCTAssertEqual(sensor.CurrentTimeStampCycle, 1.0) + } + + func testTimeSensorRejectsAnUnstampedRecordWithoutDisturbingTheNext() { + let sensor = TimeSensor() + sensor.reorderWindowTicks = windowAt51Point2Hz + + let first = sensor.calibrateTimeStamp(timeStamp: 7406506) + XCTAssertFalse(sensor.lastRecordRejected) + + // A record the firmware never stamped. The timeline holds where it was. + let unstamped = sensor.calibrateTimeStamp(timeStamp: 0) + XCTAssertTrue(sensor.lastRecordRejected) + XCTAssertEqual(unstamped, first, accuracy: 0.0) + + // And the next real sample reads as the ordinary step forward it is, + // not as a second roll-over. + let next = sensor.calibrateTimeStamp(timeStamp: 7406571) + XCTAssertFalse(sensor.lastRecordRejected) + XCTAssertEqual(next, milliseconds(7406571), accuracy: 1e-9) + XCTAssertEqual(sensor.CurrentTimeStampCycle, 0.0) + } + + /// A roll-over that lands exactly on zero is still a roll-over: what + /// separates it from an unstamped record is where its predecessor was. + func testTimeSensorStillAcceptsAGenuineRollOverOntoZero() { + let sensor = TimeSensor() + sensor.reorderWindowTicks = windowAt51Point2Hz + + _ = sensor.calibrateTimeStamp(timeStamp: 16777116) + let wrapped = sensor.calibrateTimeStamp(timeStamp: 0) + + XCTAssertFalse(sensor.lastRecordRejected) + XCTAssertEqual(wrapped, milliseconds(Double(TimestampUnwrap.ticksMax3Byte)), accuracy: 1e-9) + XCTAssertEqual(sensor.CurrentTimeStampCycle, 1.0) + } + + /// Forward motion is the default, so a roll-over preceded by a long dropout + /// is still a roll-over. A rule that defaults to "corrupt" fails here. + func testTimeSensorKeepsARollOverAfterHeavyLoss() { + let sensor = TimeSensor() + sensor.reorderWindowTicks = windowAt51Point2Hz + + _ = sensor.calibrateTimeStamp(timeStamp: 16000000) + let after = sensor.calibrateTimeStamp(timeStamp: 100) + + XCTAssertEqual(after, milliseconds(Double(TimestampUnwrap.ticksMax3Byte) + 100), + accuracy: 1e-9) + XCTAssertEqual(sensor.CurrentTimeStampCycle, 1.0) + } + + /// A reorder can land exactly on the counter's origin, which puts a host that + /// stores (unwrapped, cycle) back into the state it uses for "no sample yet". + /// The next packet is then read as a first sample and passed through, so one + /// arriving from just before the origin is placed a whole modulo late. + /// + /// Also the only case in the set whose cycle goes negative, which is a real + /// state: the raw value is derived back out of it on the next sample. + func testAReorderOntoTheOriginDoesNotLookLikeAFreshStream() { + let sensor = TimeSensor() + sensor.reorderWindowTicks = TimestampUnwrap.reorderWindowTicks( + samplingRateHz: 32768.0 / 65, maxTicks: TimestampUnwrap.ticksMax3Byte) + XCTAssertEqual(sensor.reorderWindowTicks, 520.0) + + _ = sensor.calibrateTimeStamp(timeStamp: 520) + XCTAssertEqual(sensor.calibrateTimeStamp(timeStamp: 0), 0.0, accuracy: 0.0) + XCTAssertFalse(sensor.lastRecordRejected, + "a reorder onto the origin, not an unstamped record") + + let earlier = sensor.calibrateTimeStamp( + timeStamp: Double(TimestampUnwrap.ticksMax3Byte - 16)) + XCTAssertEqual(earlier, milliseconds(-16), accuracy: 1e-9) + XCTAssertEqual(sensor.CurrentTimeStampCycle, -1.0) + } + + /// The ObjectCluster a consumer actually reads: the sensor values survive, + /// the time does not, and `timestampValid` is how a caller tells. + func testUnstampedRecordIsFlaggedOnTheObjectCluster() { + let sensor = TimeSensor() + sensor.packetIndexTimeStamp = 0 + sensor.sensorEnabled = true + sensor.reorderWindowTicks = windowAt51Point2Hz + + let good = sensor.processData(sensorPacket: [0xEA, 0x03, 0x71], + objectCluster: ObjectCluster(deviceName: "test")) + XCTAssertTrue(good.timestampValid) + + let unstamped = sensor.processData(sensorPacket: [0x00, 0x00, 0x00], + objectCluster: ObjectCluster(deviceName: "test")) + XCTAssertFalse(unstamped.timestampValid) + // The raw field is still reported as the packet sent it. + let rawIndex = unstamped.SignalNames.firstIndex { + $0.contains(Sensor.SensorFormats.Raw.rawValue) + } + XCTAssertNotNil(rawIndex) + XCTAssertEqual(unstamped.SignalData[rawIndex!], 0.0) + } +}