diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java index 9943a8481..6313f0374 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java @@ -16,6 +16,8 @@ import com.shimmerresearch.sensors.AbstractSensor.SENSORS; import com.shimmerresearch.verisense.UtilVerisenseDriver; import com.shimmerresearch.verisense.VerisenseDevice; +import com.shimmerresearch.verisense.sensors.SensorMLX90632; +import com.shimmerresearch.verisense.sensors.SensorVD6283; import com.shimmerresearch.verisense.payloaddesign.AsmBinaryFileConstants.BYTE_COUNT; import com.shimmerresearch.verisense.payloaddesign.DataBlockDetails.DATABLOCK_SENSOR_ID; import com.shimmerresearch.verisense.sensors.SensorVerisenseClock; @@ -93,13 +95,14 @@ public void parsePayloadContentsMetaData(int binFileByteIndex) throws IOExceptio // --------- End of parsing ------------------ - // The slow sensors' achieved sample rates differ from what the payload - // header can tell us (the light rate isn't stored at all and the chip adds - // per-measurement dead time; the skin-temp output cadence is refresh-code - // derived but similarly approximate), so refine them from the data itself - // before the block timings are back-filled below. - refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID.LIGHT); - refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID.SKIN_TEMP); + // The slow sensors' achieved sample rates are not what the payload header + // can tell us: the VD6283's configured rate index is not stored at all (the + // header only yields the exposure-derived upper bound, up to 10x the truth) + // and the MLX90632's output cadence is refresh-code derived and approximate. + // Refine them from the data itself - across payloads - before the block + // timings are back-filled below, and derive the CSV gap window with them. + refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID.LIGHT, SensorVD6283.NUM_SAMPLES_PER_BLOCK); + refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID.SKIN_TEMP, SensorMLX90632.NUM_SAMPLES_PER_BLOCK); // Up to, and including, payload design v10, the real-world clock time that was // stored in the payload footer was the real-world time at the end of the @@ -148,6 +151,16 @@ public void parsePayloadContentsMetaData(int binFileByteIndex) throws IOExceptio } UtilCsvSplitting.populateExpectedPayloadTsDiffLimitMapIfNeeded(verisenseDevice, verisenseDevice.getMapOfSensorIdsPerDataBlock()); + + // Now that the block timings are back-filled, measure this payload's + // slow-sensor block spacing in ABSOLUTE milliseconds and fold it into the + // running estimate the next payload will be timed with. Deliberately here + // and not next to the rate application above: the per-block stored time is + // a sub-minute tick counter, and differencing that across a payload + // boundary cannot tell a 5 s spacing from a 65 s one. + observeSlowSensorBlockSpacing(DATABLOCK_SENSOR_ID.LIGHT, SensorVD6283.NUM_SAMPLES_PER_BLOCK); + observeSlowSensorBlockSpacing(DATABLOCK_SENSOR_ID.SKIN_TEMP, SensorMLX90632.NUM_SAMPLES_PER_BLOCK); + calculateAndSetPayloadPackagingDelayMs(); } @@ -316,17 +329,247 @@ private boolean isParserAtEndOfBuffer(int bufferLength, int currentByteIndex) { } /** - * Derive a slow sensor's (ambient light / skin temp) achieved sample period - * from the spacing of consecutive same-sensor block end ticks and apply it to - * those blocks' sampling rate before their timings are back-filled. The - * header-derived rates are only estimates (the light rate isn't stored at all - * - see SensorVD6283.getRateFreq - and the skin-temp cadence is refresh-code - * derived), and each block holds a fixed number of samples, so - * {@code inter-block ticks / samples-per-block} is the exact per-sample period. - * With fewer than two blocks in the payload the header-derived estimate the - * blocks were created with is left in place. + * Refine a slow sensor's achieved per-sample period from the data and + * (re)derive its CSV gap-splitting window, choosing between the cross-payload + * treatment DEV-979 added, the per-payload one that came before it, and - for + * the MLX90632 at a slow enough rate - a window seeded straight from the + * header. + *

+ * A block's stored end time is a counter that wraps every minute, so + * differencing two blocks' end ticks is only sound while the true spacing is + * under a minute. {@link UtilCsvSplitting#isSlowSensorSpanUnambiguousAcrossPayloads(DATABLOCK_SENSOR_ID, int)} + * asks whether that holds for the sensor's SLOWEST legitimate rate: the VD6283 + * qualifies (a 10-sample block spans at most 20 s) and needs the cross-payload + * measurement because its configured rate is not in the payload at all. + *

+ * The MLX90632 fails that gate on its slowest rates, but its refresh code IS + * in the payload, so its ACTUAL configured output rate is known + * ({@link com.shimmerresearch.verisense.sensors.SensorMLX90632#getRateFreq()}, + * the value written to the CSV "Configured" line): + *

+ * + * @param slowSensorId the slow sensor's data block id + * @param samplesPerBlock the sensor's fixed samples per block + */ + void refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID slowSensorId, int samplesPerBlock) { + if(UtilCsvSplitting.isSlowSensorSpanUnambiguousAcrossPayloads(slowSensorId, samplesPerBlock)) { + refineSlowSensorSamplingRateAcrossPayloads(slowSensorId, samplesPerBlock); + } else if(slowSensorId==DATABLOCK_SENSOR_ID.SKIN_TEMP + && !isSkinTempBlockSpanUnderAMinute(samplesPerBlock)) { + UtilCsvSplitting.refineSlowSensorGapWindow(verisenseDevice, slowSensorId, samplesPerBlock); + } else { + refineSlowSensorSamplingRatePerPayload(slowSensorId); + } + } + + /** + * Whether the MLX90632's ACTUAL configured output rate (from the refresh code + * in the payload header) puts a full block's span under the one-minute + * end-tick wrap - i.e. whether the per-payload tick-differencing path is safe + * for this recording. Unknown/zero rate is treated as NOT under a minute (the + * safe direction: use the header-seeded window, not the tick path). + * + * @param samplesPerBlock the MLX90632's fixed samples per block + * @return true when {@code samplesPerBlock / configuredOutputRateHz < 60 s} + */ + private boolean isSkinTempBlockSpanUnderAMinute(int samplesPerBlock) { + double configuredRateHz = verisenseDevice.getSamplingRateForSensor(SENSORS.MLX90632); + if(!(configuredRateHz > 0)) { + return false; + } + double blockSpanS = samplesPerBlock / configuredRateHz; + return blockSpanS < (AsmBinaryFileConstants.TICKS_PER_MINUTE / AsmBinaryFileConstants.TICKS_PER_SECOND); + } + + /** + * Apply a slow sensor's measured per-sample period to this payload's blocks + * before their timings are back-filled, and (re)derive its CSV gap-splitting + * window - the cross-payload path DEV-979 added, for sensors that pass + * {@link UtilCsvSplitting#isSlowSensorSpanUnambiguousAcrossPayloads(DATABLOCK_SENSOR_ID, int)}. + *

+ * The period comes from the running estimate built by + * {@link #observeSlowSensorBlockSpacing(DATABLOCK_SENSOR_ID, int)}, which runs + * at the END of the parse - see the note on ordering there. Measuring it at + * all is the only way to recover the VD6283's rate: the configured rate index + * is operational-config byte 75 and is NOT copied into the payload header, so + * the header-derived value the blocks are created with is merely the + * exposure-limited UPPER BOUND (see + * {@link com.shimmerresearch.verisense.sensors.SensorVD6283#getRateFreq()}). + * With the firmware's default 1 Hz rate and the default 100 ms exposure that + * bound is ten times the truth, which compressed every 10-sample block into + * 0.9 s of the 10 s it actually spans and left the remaining 9.1 s looking + * like a gap - splitting the CSV on every single block (DEV-979). + *

+ * RESIDUAL, not fixed here: the first TWO blocks of each CSV set are timed + * before a boundary between them has been observed, so they keep the + * header-derived estimate. Their samples are laid out over + * {@code (N-1) x estimatedPeriod} instead of {@code (N-1) x truePeriod}, which + * for a 10-sample light block at the default exposure puts the block's - and + * therefore the CSV header's - reported start time 8.1 s late at 1 Hz. + * Re-timing them would mean revisiting a block after the next one arrives, by + * which point the file parser has deep-cloned it into the CSV dataset it is + * accumulating, so the fix does not belong in the driver. The sample VALUES + * and the block end times are unaffected. + * + * @param slowSensorId the slow sensor's data block id + * @param samplesPerBlock the sensor's fixed samples per block + */ + void refineSlowSensorSamplingRateAcrossPayloads(DATABLOCK_SENSOR_ID slowSensorId, int samplesPerBlock) { + boolean payloadCarriesThisSensor = false; + for(DataBlockDetails dataBlockDetails:listOfDataBlocksInOrder) { + if(dataBlockDetails.datablockSensorId==slowSensorId) { + payloadCarriesThisSensor = true; + break; + } + } + if(!payloadCarriesThisSensor) { + return; + } + + double medianPeriodS = UtilCsvSplitting.recordAndGetSlowSensorPeriodS(verisenseDevice, slowSensorId, Double.NaN); + if(UtilCsvSplitting.isSlowSensorPeriodPlausible(verisenseDevice, slowSensorId, medianPeriodS)) { + double achievedRateHz = 1.0/medianPeriodS; + for(DataBlockDetails dataBlockDetails:listOfDataBlocksInOrder) { + if(dataBlockDetails.datablockSensorId==slowSensorId) { + dataBlockDetails.setSamplingRate(achievedRateHz); + dataBlockDetails.calculateTimestampDiffInS(); + } + } + } + + UtilCsvSplitting.refineSlowSensorGapWindow(verisenseDevice, slowSensorId, samplesPerBlock); + } + + /** + * Measure a slow sensor's achieved per-sample period from the spacing of + * consecutive same-sensor block END TIMES, across payload boundaries, and feed + * it into the running estimate the next payload will be timed with. + *

+ * Each block holds a fixed number of samples and is stamped with the time of + * its LAST sample, so {@code inter-block time / samples-per-block} is the + * achieved per-sample period - the technique the storage-format spec + * prescribes for the LSM6DSV. + *

+ * ORDERING, and why this is separate from + * {@link #refineSlowSensorSamplingRateAcrossPayloads(DATABLOCK_SENSOR_ID, int)}: + * this runs at the END of the payload parse, once the block timings have been + * back-filled, so it can difference ABSOLUTE real-world-clock milliseconds - + * the very quantity + * {@link UtilCsvSplitting#isDataBlockContinuous(SENSORS, DataSegmentDetails, DataBlockDetails)} + * judges a boundary on. The per-block stored time is only a SUB-MINUTE tick + * counter, so differencing that across payloads cannot tell a 5 s spacing from + * a 65 s one: a real 65 s gap between 10-sample light blocks aliases to 0.5 s + * per sample, i.e. 2 Hz, which is a legitimate firmware rate and so passes any + * plausibility test. Detection was never affected (the continuity check works + * in absolute ms) but the aliased value would have been learned and applied, + * mis-timing the blocks after a real gap until the state was next cleared. + * Absolute milliseconds remove the ambiguity at the source rather than + * guarding against it downstream. + *

+ * The in-payload path used by sensors that fail the unambiguous-span gate + * keeps differencing ticks, which is safe there: two blocks in one payload are + * at most a payload duration apart. + * + * @param slowSensorId the slow sensor's data block id + * @param samplesPerBlock the sensor's fixed samples per block */ - private void refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID slowSensorId) { + void observeSlowSensorBlockSpacing(DATABLOCK_SENSOR_ID slowSensorId, int samplesPerBlock) { + if(!UtilCsvSplitting.isSlowSensorSpanUnambiguousAcrossPayloads(slowSensorId, samplesPerBlock)) { + return; + } + + // Only WHOLE blocks may be measured. A block that + // splitDataBlocksAtMiddayMidnight cut in two would otherwise read as an + // extra boundary a fraction of a second wide carrying a reduced sample + // count, so a split block is measured on its SECOND part - which keeps the + // original block's end time, splitAndStartAtSampleIndex only moves the + // start - with the two parts' sample counts added back together. + List slowSensorBlocks = new ArrayList(); + List wholeBlockSampleCounts = new ArrayList(); + int pendingFirstPartSampleCount = 0; + for(DataBlockDetails dataBlockDetails:listOfDataBlocksInOrder) { + if(dataBlockDetails.datablockSensorId!=slowSensorId) { + continue; + } + if(dataBlockDetails.isFirstPartOfSplitDataBlock()) { + pendingFirstPartSampleCount = dataBlockDetails.getSampleCount(); + continue; + } + slowSensorBlocks.add(dataBlockDetails); + wholeBlockSampleCounts.add(Integer.valueOf(dataBlockDetails.getSampleCount()+pendingFirstPartSampleCount)); + pendingFirstPartSampleCount = 0; + } + if(slowSensorBlocks.isEmpty()) { + return; + } + + // This runs before PayloadContentsDetails sorts the payload by continuity, + // so listOfDataBlocksInOrder is still in file - i.e. temporal - order and + // the last block of the sensor is genuinely its latest. + Double previousBlockEndTimeMs = UtilCsvSplitting.SLOW_SENSOR_LAST_BLOCK_END_TIME_RWC_MS.get(slowSensorId); + for(int i=0;i0) { + double observedPeriodS = ((blockEndTimeMs-previousBlockEndTimeMs.doubleValue())/1000)/sampleCount; + UtilCsvSplitting.recordAndGetSlowSensorPeriodS(verisenseDevice, slowSensorId, observedPeriodS); + } + previousBlockEndTimeMs = Double.valueOf(blockEndTimeMs); + } + if(previousBlockEndTimeMs!=null) { + UtilCsvSplitting.SLOW_SENSOR_LAST_BLOCK_END_TIME_RWC_MS.put(slowSensorId, previousBlockEndTimeMs); + } else { + UtilCsvSplitting.SLOW_SENSOR_LAST_BLOCK_END_TIME_RWC_MS.remove(slowSensorId); + } + + UtilCsvSplitting.refineSlowSensorGapWindow(verisenseDevice, slowSensorId, samplesPerBlock); + } + + /** + * The PER-PAYLOAD refinement as it stood before DEV-979, body unchanged from + * master 6d27fb2 (including the {@code size()/2} upper-middle median and the + * early return below two blocks). Reached for a slow sensor that fails + * {@link UtilCsvSplitting#isSlowSensorSpanUnambiguousAcrossPayloads(DATABLOCK_SENSOR_ID, int)} + * AND whose block genuinely spans under a minute - i.e. the MLX90632 at the + * DEV-927 16 Hz configuration (a 16-sample block spans ~1 s), where the + * sub-minute tick delta is unambiguous. The MLX90632 at 0.25 Hz output (a + * 16-sample block spans ~64 s) is routed away from here by + * {@link #refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID, int)} + * because its tick delta across a payload boundary WOULD alias. + *

+ * Byte-identity for the DEV-927 skin temp holds by construction: the applied + * period is still this payload's own median, not a whole-file one, and the + * window is still seeded with the old formula. The DEV-927 reference CSVs + * (ASM_PC Test_065) cannot be reached from this environment, so nothing about + * that sensor's timing is changed on trust. + * + * @param slowSensorId the slow sensor's data block id + */ + void refineSlowSensorSamplingRatePerPayload(DATABLOCK_SENSOR_ID slowSensorId) { List slowSensorBlocks = new ArrayList(); for(DataBlockDetails dataBlockDetails:listOfDataBlocksInOrder) { if(dataBlockDetails.datablockSensorId==slowSensorId) { @@ -407,6 +650,7 @@ private void refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID slow } } + private void backfillDataBlockRwcTimestamps() { backfillDataBlockUcClockOrRwcTimestamps(false); } diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java index f6c23ee08..9f882103a 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java @@ -1,5 +1,7 @@ package com.shimmerresearch.verisense.payloaddesign; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -8,6 +10,8 @@ import com.shimmerresearch.verisense.UtilVerisenseDriver; import com.shimmerresearch.verisense.VerisenseDevice; import com.shimmerresearch.verisense.payloaddesign.DataBlockDetails.DATABLOCK_SENSOR_ID; +import com.shimmerresearch.verisense.sensors.SensorMLX90632; +import com.shimmerresearch.verisense.sensors.SensorVD6283; public class UtilCsvSplitting { @@ -20,18 +24,75 @@ public class FILE_GAP_TOLERANCE_MULTIPLIER { * inter-block gap, as a multiple of the achieved median block spacing, that * is still treated as continuous. The MLX90632's conversions can slip by * several refresh periods and then catch up (observed up to +12.5% block - * spacing on the DEV-927 validation recording with no samples lost), and the - * window is seeded from the first payload that carries >= 2 blocks - often a - * single inter-block gap, i.e. no spread information - so the standard - * LOWER (-10%) band is routinely violated by healthy data. A genuinely - * dropped block doubles the spacing (2x), so 1.5x keeps comfortable margin - * on both sides. + * spacing on the DEV-927 validation recording with no samples lost) and the + * VD6283's cadence is bimodal (exposure vs exposure + dead time), so the + * standard LOWER (-10%) band is routinely violated by healthy data. A + * genuinely dropped block doubles the spacing (2x), so 1.5x keeps + * comfortable margin on both sides. */ public static final double SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO = 1.5; + /** + * MLX90632 only: how far a SINGLE block boundary's apparent rate may sit + * either side of the configured output rate and still be plausible, used to + * widen that rate into the a-priori window applied before any boundary has + * been measured. + *

+ * The chip's conversions slip by several refresh periods and then catch up + * (+12.5% block spacing observed on the DEV-927 validation recording, with + * no samples lost), so a boundary reads ~12.5% slow and the one after it + * correspondingly fast. Only the AVERAGE rate is bounded by the configured + * one; a single boundary is not, which is why the configured rate alone is + * too tight a bound. 1.15 covers the observed slip with a little margin. + *

+ * Not used for the VD6283, whose a-priori bounds come from the firmware's + * rate table instead - its configured rate is not in the payload at all, + * and its sampling is a plain periodic timer with no slip-and-catch-up + * behaviour (a failed read costs a whole period, it never shortens one). + */ + public static final double SLOW_SENSOR_CONVERSION_SLIP_TOLERANCE = 1.15; } + + /** + * Slow sensors only: how many block boundaries must have been observed before + * the MEASURED window replaces the provisional one. Below this the median is + * not an estimate of anything - the boundary about to be judged is itself one + * of the one or two values it would be built from, so it would always be found + * continuous, and the first boundary of every CSV set would be unreportable no + * matter how large its gap. + */ + public static final int SLOW_SENSOR_MIN_OBSERVATIONS_FOR_MEASURED_WINDOW = 3; + + /** + * Slow sensors only: how many of the most recently observed per-sample periods + * {@link #recordAndGetSlowSensorPeriodS(VerisenseDevice, DATABLOCK_SENSOR_ID, double)} keeps + * per sensor. Bounded so the estimate follows genuine long-term drift instead + * of averaging a multi-day recording, and so the median stays cheap to re-take + * on every payload. Large enough that occasional dropped blocks cannot move + * the median once the history is full - but see the design note on + * {@link #refineSlowSensorGapWindow(VerisenseDevice, DATABLOCK_SENSOR_ID, int)} + * for what SUSTAINED loss does. + */ + public static final int SLOW_SENSOR_OBSERVED_RATE_HISTORY_MAX = 256; protected static HashMap SAMPLING_RATE_LIMITS_PER_SENSOR = new HashMap(); + /** + * Slow sensors only: the ABSOLUTE real-world-clock end time (ms) of the last + * block seen for each slow-sensor data block id, carried from one payload to + * the next so the achieved per-sample period can be measured across payload + * boundaries. A 1 Hz light block spans 10 s while a payload spans ~2 s, so a + * payload carries at most one light block and there is no inter-block gap + * inside it to measure. + *

+ * Absolute milliseconds, not the per-block sub-minute tick counter, because a + * tick delta cannot tell a 5 s spacing from a 65 s one - a real 65 s gap + * between 10-sample light blocks aliases to a perfectly legitimate 2 Hz. + * Populated only once the block timings have been back-filled. + */ + protected static HashMap SLOW_SENSOR_LAST_BLOCK_END_TIME_RWC_MS = new HashMap(); + + protected static HashMap> SLOW_SENSOR_OBSERVED_BLOCK_PERIODS_S = new HashMap>(); + public static boolean isTsDifferenceOutsideOfLimits(double expectedPayloadTsDiffLimits[], double unixTimeInMs_1, double unixTimeInMs_2) { double differenceInMillisec = Math.abs(unixTimeInMs_1 - unixTimeInMs_2); if(differenceInMillisec < expectedPayloadTsDiffLimits[0] || differenceInMillisec > expectedPayloadTsDiffLimits[1]) { @@ -99,10 +160,291 @@ public static double[] calculateSamplingRateLimits(double configuredSamplingRate return new double[] {configuredSamplingRate*FILE_GAP_TOLERANCE_MULTIPLIER.LOWER, configuredSamplingRate*FILE_GAP_TOLERANCE_MULTIPLIER.UPPER}; } + /** + * Clears everything the CSV-splitting windows are derived from. Called + * whenever a whole CSV set is written out (end of file, config change or + * device reset) so that no measurement leaks across a CSV-set boundary - the + * timing regime either side of a reset is unrelated. + */ public static void clearMapOfSamplingRateLimitsPerSensor() { SAMPLING_RATE_LIMITS_PER_SENSOR.clear(); + SLOW_SENSOR_LAST_BLOCK_END_TIME_RWC_MS.clear(); + SLOW_SENSOR_OBSERVED_BLOCK_PERIODS_S.clear(); } - + + /** + * Records one observed slow-sensor per-sample period and returns the sensor's + * best current estimate of it. + *

+ * The slow sensors (VD6283 ambient light, MLX90632 skin temp) buffer a fixed + * number of samples and emit the block only once it is full, stamping it with + * the time of its LAST sample. The spacing between two consecutive blocks' + * end times divided by the samples per block is therefore the achieved + * per-sample period - the same technique the storage-format spec prescribes for + * the LSM6DSV, and the only way to recover the VD6283's rate at all, because + * the configured rate index is not stored in the payload (see + * {@link com.shimmerresearch.verisense.sensors.SensorVD6283#getRateFreq()}). + *

+ * The MEDIAN over the accumulated history is returned rather than the latest + * delta. A single failed I2C read makes one block take an extra period to fill + * without losing a sample slot (hal_slowSensorSampler.c increments the count + * only on a successful read), and a dropped block doubles the delta - taking + * the raw delta would stretch that block's samples by 10% or 100%, whereas the + * median keeps every block on the true period, which is where the samples + * actually are. The history is bounded to + * {@link #SLOW_SENSOR_OBSERVED_RATE_HISTORY_MAX} so the estimate still follows + * genuine long-term drift. + * + * @param slowSensorId the slow sensor's data block id + * @param observedPeriodS the period just measured, or NaN to only read the estimate back + * @return the median observed period in seconds, or NaN if nothing has been observed + */ + public static double recordAndGetSlowSensorPeriodS(VerisenseDevice verisenseDevice, DATABLOCK_SENSOR_ID slowSensorId, double observedPeriodS) { + List observedPeriodsS = SLOW_SENSOR_OBSERVED_BLOCK_PERIODS_S.get(slowSensorId); + if(observedPeriodsS==null) { + observedPeriodsS = new ArrayList(); + SLOW_SENSOR_OBSERVED_BLOCK_PERIODS_S.put(slowSensorId, observedPeriodsS); + } + // Every finite positive observation is learned from, anomalies included. + // Filtering here cannot be done safely: the history is empty after every + // clear, so the first observation would define what counts as plausible and a + // CSV set opening on a dropped block would lock the estimate onto the wrong + // period for good. Anomalies are handled by the median, and by the window + // builder rejecting implausible values when it picks the fast side. + if(isSlowSensorPeriodPlausible(verisenseDevice, slowSensorId, observedPeriodS)) { + observedPeriodsS.add(Double.valueOf(observedPeriodS)); + if(observedPeriodsS.size()>SLOW_SENSOR_OBSERVED_RATE_HISTORY_MAX) { + // Keep the NEWEST observations - the estimate tracks the sensor. + observedPeriodsS.subList(0, observedPeriodsS.size()-SLOW_SENSOR_OBSERVED_RATE_HISTORY_MAX).clear(); + } + } + return calculateMedian(observedPeriodsS); + } + + /** + * Whether an observed per-sample period is one the sensor could actually have + * produced, i.e. finite, positive and inside + * {@code getSlowSensorPlausibleRateRangeHz}. + *

+ * This catches a period no configuration could have produced - a dropped block + * or a clock correction stretching a boundary well past the slowest rate, for + * instance - so that neither the running estimate nor the timing of a block + * can be built from one. It does NOT and cannot catch tick aliasing: a real + * 65 s gap between 10-sample light blocks aliases to 0.5 s per sample, i.e. + * 2 Hz, which IS a legitimate firmware rate. That is why the cross-payload + * measurement differences absolute real-world-clock milliseconds instead of + * ticks (see + * {@code PayloadContentsDetailsV8orAbove.observeSlowSensorBlockSpacing}) - the + * ambiguity is removed at the source rather than filtered here. + * + * @param verisenseDevice the device being parsed + * @param slowSensorId the slow sensor's data block id + * @param observedPeriodS the candidate per-sample period in seconds + * @return true when the period is one the sensor could have produced + */ + public static boolean isSlowSensorPeriodPlausible(VerisenseDevice verisenseDevice, DATABLOCK_SENSOR_ID slowSensorId, double observedPeriodS) { + if(Double.isNaN(observedPeriodS) || Double.isInfinite(observedPeriodS) || !(observedPeriodS>0)) { + return false; + } + double[] plausibleRateRangeHz = getSlowSensorPlausibleRateRangeHz(verisenseDevice, slowSensorId); + if(plausibleRateRangeHz==null) { + return true; + } + double observedRateHz = 1.0/observedPeriodS; + return observedRateHz>=plausibleRateRangeHz[0] && observedRateHz<=plausibleRateRangeHz[1]; + } + + public static int getSlowSensorObservationCount(DATABLOCK_SENSOR_ID slowSensorId) { + List observedPeriodsS = SLOW_SENSOR_OBSERVED_BLOCK_PERIODS_S.get(slowSensorId); + return observedPeriodsS==null? 0:observedPeriodsS.size(); + } + + /** + * (Re)derives a slow sensor's CSV gap-splitting window. + *

+ * Once the achieved per-sample period is known the window is the #285 formula + * over the measured history: gap side + * {@code median / SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO} so healthy jitter + * stays continuous while a dropped block (2x spacing) is reported, fast side + * the quickest boundary that is still plausible against the median, with the + * standard tolerance. The fast side has to be an extremum rather than the + * median because the MLX90632's conversions slip and then catch up (+12.5% + * observed on the DEV-927 recording, so the catch-up boundary reads + * correspondingly fast); excluding the implausible ones here rather than when + * they were learned stops an overlap artefact widening the fast side past + * itself while still letting the median recover from an early anomaly. + *

+ * Below {@link #SLOW_SENSOR_MIN_OBSERVATIONS_FOR_MEASURED_WINDOW} + * observations - which includes the first block of every CSV set, when there + * are none - an A-PRIORI window is used instead, because the boundary being + * judged is itself one of the one or two values a measured window would be + * built from, so that window would simply re-centre on whatever it was about + * to judge and the first boundary of every CSV set would be continuous no + * matter how large its gap. The a-priori bounds come from what the hardware + * can actually do, not from an arbitrary multiple: see + * {@code getSlowSensorPlausibleRateRangeHz}. + *

+ * DESIGN NOTE: the window follows the data, so SUSTAINED block loss is + * eventually learned as the cadence. If every other block went missing for + * more than {@link #SLOW_SENSOR_OBSERVED_RATE_HISTORY_MAX} boundaries the + * median would move onto the halved rate and the loss would stop being + * reported; the return to the true cadence is then reported once, as a single + * split. That is the price of tracking a rate the payload does not carry. + * + * @param verisenseDevice the device being parsed + * @param slowSensorId the slow sensor's data block id + * @param samplesPerBlock the sensor's fixed samples per block + */ + public static void refineSlowSensorGapWindow(VerisenseDevice verisenseDevice, DATABLOCK_SENSOR_ID slowSensorId, int samplesPerBlock) { + List observedPeriodsS = SLOW_SENSOR_OBSERVED_BLOCK_PERIODS_S.get(slowSensorId); + double[] samplingRateLimits = null; + if(observedPeriodsS!=null && observedPeriodsS.size()>=SLOW_SENSOR_MIN_OBSERVATIONS_FOR_MEASURED_WINDOW) { + double medianRateHz = 1.0/calculateMedian(observedPeriodsS); + if(medianRateHz>0 && !Double.isInfinite(medianRateHz)) { + double plausibleRateCeilingHz = medianRateHz*FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO; + double fastestPlausibleRateHz = medianRateHz; + for(Double observedPeriodS:observedPeriodsS) { + double observedRateHz = 1.0/observedPeriodS.doubleValue(); + if(observedRateHz>fastestPlausibleRateHz && observedRateHz<=plausibleRateCeilingHz) { + fastestPlausibleRateHz = observedRateHz; + } + } + samplingRateLimits = new double[] { + medianRateHz/FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO, + fastestPlausibleRateHz*FILE_GAP_TOLERANCE_MULTIPLIER.UPPER}; + } + } + if(samplingRateLimits==null) { + double[] plausibleRateRangeHz = getSlowSensorPlausibleRateRangeHz(verisenseDevice, slowSensorId); + if(plausibleRateRangeHz==null) { + return; + } + samplingRateLimits = new double[] { + plausibleRateRangeHz[0]/FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO, + plausibleRateRangeHz[1]*FILE_GAP_TOLERANCE_MULTIPLIER.UPPER}; + } + // Deliberately unconditional: populateExpectedPayloadTsDiffLimitMapIfNeeded + // would otherwise leave a band built from the configured rate in place, and + // for the VD6283 that "configured rate" is only an exposure-derived upper + // bound which can be 10x the truth. + for(SENSORS sensorClassKey:verisenseDevice.getOrCreateListOfSensorClassKeysForDataBlockId(slowSensorId)) { + if(sensorClassKey!=SENSORS.CLOCK) { + SAMPLING_RATE_LIMITS_PER_SENSOR.put(sensorClassKey, samplingRateLimits); + } + } + } + + /** + * Whether a slow sensor's block spacing can be measured ACROSS payloads from + * the sub-minute tick counter without ambiguity. + *

+ * A block's stored end time is a counter that wraps every minute, so a delta + * between two blocks is only recoverable (by re-basing a negative delta by one + * minute) while the true spacing is under a minute. Inside one payload that is + * guaranteed by the payload's own duration, but across payloads it has to be + * bounded by what the sensor could be configured to do: the slowest rate the + * hardware offers times the samples per block. + *

+ * A 10-sample VD6283 block spans at most 20 s (slowest firmware rate 0.5 Hz) + * and is always safe. A 16-sample MLX90632 block spans 64 s in the common + * medical-mode worst case (0.5 Hz refresh / 2 = 0.25 Hz output) and 96 s in + * the extended-mode worst case + * ({@link com.shimmerresearch.verisense.sensors.SensorMLX90632#MIN_OUTPUT_RATE_HZ}, + * 0.5 Hz refresh / 3 = 0.167 Hz), and this method uses that worst case - both + * exceed the 60 s unambiguous span, so the skin temp never qualifies. It costs + * nothing: the MLX90632's refresh code IS stored in the payload, so its + * header-derived rate is already correct and it only needs the within-payload + * refinement it has always had. + * + * @param slowSensorId the slow sensor's data block id + * @param samplesPerBlock the sensor's fixed samples per block + * @return true when a cross-payload tick delta is unambiguous + */ + public static boolean isSlowSensorSpanUnambiguousAcrossPayloads(DATABLOCK_SENSOR_ID slowSensorId, int samplesPerBlock) { + double[] plausibleRateRangeHz = null; + if(slowSensorId==DATABLOCK_SENSOR_ID.LIGHT) { + plausibleRateRangeHz = new double[] {SensorVD6283.MIN_SAMPLE_RATE_HZ, SensorVD6283.MAX_SAMPLE_RATE_HZ}; + } else if(slowSensorId==DATABLOCK_SENSOR_ID.SKIN_TEMP) { + plausibleRateRangeHz = new double[] {SensorMLX90632.MIN_OUTPUT_RATE_HZ, SensorMLX90632.MAX_OUTPUT_RATE_HZ}; + } + if(plausibleRateRangeHz==null || !(plausibleRateRangeHz[0]>0)) { + return false; + } + double maximumBlockSpanS = samplesPerBlock/plausibleRateRangeHz[0]; + return maximumBlockSpanS<(AsmBinaryFileConstants.TICKS_PER_MINUTE/AsmBinaryFileConstants.TICKS_PER_SECOND); + } + + /** + * The {min, max} per-sample rate a slow sensor could legitimately be running + * at, used for the a-priori gap window before anything has been measured. + *

+ * VD6283: the firmware's whole rate table + * ({@link com.shimmerresearch.verisense.sensors.SensorVD6283#MIN_SAMPLE_RATE_HZ} + * ..{@link com.shimmerresearch.verisense.sensors.SensorVD6283#MAX_SAMPLE_RATE_HZ} + * = 0.5..20 Hz), because the configured index is not in the payload and the + * exposure only bounds the rate from above. A 10-sample block may therefore + * legitimately span anything from 0.5 s to 20 s. + *

+ * BLIND SPOT, quantified: with the standard tolerances that window is + * [0.33, 22] Hz, and a boundary presents {@code 10 / deltaS}, so on the first + * two boundaries of a CSV set any spacing up to 30 s is accepted. At the + * firmware's default 1 Hz that means up to 20 s of genuinely lost light data + * goes unreported there, permanently - those two boundaries are never + * re-judged. The fast side likewise accepts a backwards clock jump of up to + * ~9.5 s. A 60 s spacing does split. This is the price of not knowing the + * configured rate: the alternative, centring the window on one or two + * observations, cannot report anything at all (the boundary being judged is + * the estimate). From the third boundary on the measured window applies and + * the tolerance is 1.5x the achieved period. + *

+ * MLX90632: the refresh code IS stored in the payload, so the configured + * output rate is known; it is widened by + * {@link FILE_GAP_TOLERANCE_MULTIPLIER#SLOW_SENSOR_CONVERSION_SLIP_TOLERANCE} + * to cover the documented conversion slip and catch-up, and falls back to the + * refresh table's full span if the configured rate is unusable. + * + * @param verisenseDevice the device being parsed + * @param slowSensorId the slow sensor's data block id + * @return the {min, max} plausible rate in Hz, or null if it cannot be bounded + */ + public static double[] getSlowSensorPlausibleRateRangeHz(VerisenseDevice verisenseDevice, DATABLOCK_SENSOR_ID slowSensorId) { + if(slowSensorId==DATABLOCK_SENSOR_ID.LIGHT) { + return new double[] {SensorVD6283.MIN_SAMPLE_RATE_HZ, SensorVD6283.MAX_SAMPLE_RATE_HZ}; + } + if(slowSensorId==DATABLOCK_SENSOR_ID.SKIN_TEMP) { + double configuredSamplingRate = verisenseDevice.getSamplingRateForSensor(SENSORS.MLX90632); + if(configuredSamplingRate>0 && !Double.isNaN(configuredSamplingRate) && !Double.isInfinite(configuredSamplingRate)) { + return new double[] { + configuredSamplingRate/FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_CONVERSION_SLIP_TOLERANCE, + configuredSamplingRate*FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_CONVERSION_SLIP_TOLERANCE}; + } + return new double[] {SensorMLX90632.MIN_OUTPUT_RATE_HZ, SensorMLX90632.MAX_OUTPUT_RATE_HZ}; + } + return null; + } + + /** + * Median of a list of observations - the mean of the two middle values for an + * even-sized input, so that neither of a pair straddling the middle can hand + * the estimate to an outlier on its own. Sorts a copy; the caller's list is + * kept in arrival order so its oldest entries can be trimmed. + * + * @param values the observations, in arrival order + * @return the median, or NaN when there are none + */ + public static double calculateMedian(List values) { + if(values==null || values.isEmpty()) { + return Double.NaN; + } + List sortedValues = new ArrayList(values); + Collections.sort(sortedValues); + int middleIndex = sortedValues.size()/2; + if(sortedValues.size()%2==0) { + return (sortedValues.get(middleIndex-1).doubleValue()+sortedValues.get(middleIndex).doubleValue())/2.0; + } + return sortedValues.get(middleIndex).doubleValue(); + } + public static String isDataBlockContinuous(SENSORS sensorClassKey, DataSegmentDetails dataSegmentDetailsPrevious, DataBlockDetails nextDataBlockDetails) { //Get last data block from existing dataset DataBlockDetails previousDataBlockDetails = dataSegmentDetailsPrevious.getListOfDataBlocks().get(dataSegmentDetailsPrevious.getDataBlockCount()-1); diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorMLX90632.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorMLX90632.java index edcf0cd56..8bcbf4fab 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorMLX90632.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorMLX90632.java @@ -51,6 +51,20 @@ public class SensorMLX90632 extends AbstractSensor { /** Refresh-rate code (header byte 32 bits 3:1) -> chip refresh Hz. */ public static final double[] REFRESH_HZ_TABLE = {0.5, 1, 2, 4, 8, 16, 32, 64}; + /** + * Output-rate bounds implied by the refresh table across BOTH modes: the + * slowest configuration is REFRESH_HZ_TABLE[0] (0.5 Hz) divided by + * {@link #SUB_MEASUREMENTS_EXTENDED} (3) = 0.167 Hz, and the fastest is + * REFRESH_HZ_TABLE[7] (64 Hz) divided by {@link #SUB_MEASUREMENTS_MEDICAL} + * (2) = 32 Hz. A parser can therefore bound this sensor's output rate from + * the payload header before it has measured anything - and unlike the + * VD6283's, the rate itself IS recoverable from the header (the refresh code + * is stored), so {@link #getRateFreq()} is a real estimate rather than only an + * upper bound. + */ + public static final double MIN_OUTPUT_RATE_HZ = 0.5/3; + public static final double MAX_OUTPUT_RATE_HZ = 32.0; + /** Sub-measurements per output: medical mode = 2, extended mode = 3. */ public static final int SUB_MEASUREMENTS_MEDICAL = 2; public static final int SUB_MEASUREMENTS_EXTENDED = 3; diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorVD6283.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorVD6283.java index 37ec8437b..1932de191 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorVD6283.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorVD6283.java @@ -81,7 +81,16 @@ public class SensorVD6283 extends AbstractSensor { {-0.028752, 0.506372, -0.120614}, {-0.552625, 0.335866, 0.494781}}; - /** Poll ceiling in continuous mode (firmware slow-sensor sampler). */ + /** + * The rates the firmware can actually be configured to, from the slow-sensor + * sampler's index table (hal_slowSensorSampler.c + * {@code slowSensorRateMs[] = {0, 2000, 1000, 500, 200, 100, 50}}, i.e. + * 0/0.5/1/2/5/10/20 Hz). The index lives in operational-config byte 75 + * (LIGHT_SAMPLE_RATE_INDEX) and is NOT copied into the stored payload header, + * so a file parser cannot read the configured rate back - it can only bound + * it. MAX_SAMPLE_RATE_HZ doubles as the poll ceiling in continuous mode. + */ + public static final double MIN_SAMPLE_RATE_HZ = 0.5; public static final double MAX_SAMPLE_RATE_HZ = 20.0; public static final String UNITS_LUX = "lux"; @@ -307,10 +316,28 @@ public boolean isDarkChannelEnabled() { } /** - * Exposure-limited sample-rate ESTIMATE (Hz). The configured rate is not in - * the stored payload header and the chip adds per-measurement dead time, so - * this only seeds data-block timing - the parser refines the rate per payload - * from consecutive light-block timestamps. + * Exposure-derived UPPER BOUND on the sample rate (Hz), not the configured + * rate. + *

+ * The exposure only caps how fast the chip can measure: the firmware sets the + * inter-measurement time to the configured sample period and the VD6283 + * measures every {@code max(inter-measurement, exposure)}, so the exposure + * bounds the rate from above and says nothing about it otherwise. The rate + * ITSELF is operational-config byte 75 (LIGHT_SAMPLE_RATE_INDEX) into + * {@link #MIN_SAMPLE_RATE_HZ}..{@link #MAX_SAMPLE_RATE_HZ}, which is NOT + * stored in the payload header - the firmware defaults it to 1 Hz when the + * sensor is enabled with index 0 (ASM_Production/main.c), and 1 Hz with the + * default 100 ms exposure is a factor of TEN below this bound. + *

+ * So this value is only good enough to seed the timing of the FIRST block of a + * CSV set; every block after it is re-timed from the measured inter-block + * spacing by + * {@code PayloadContentsDetailsV8orAbove.refineSlowSensorSamplingRateFromBlockTicks}. + * Deriving sample spacing from it alone compressed each 10-sample block into + * 0.9 s of a 10 s span and left the remainder looking like a 9.1 s gap, which + * split the CSV on every block (DEV-979). + * + * @return the exposure-limited upper bound on the sample rate in Hz */ public double getRateFreq() { return Math.min(MAX_SAMPLE_RATE_HZ, 1e6 / getExposureUs()); diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/verisense/payloaddesign/API_00009_VerisenseSlowSensorGapWindow.java b/ShimmerDriver/src/test/java/com/shimmerresearch/verisense/payloaddesign/API_00009_VerisenseSlowSensorGapWindow.java new file mode 100644 index 000000000..84cc727ec --- /dev/null +++ b/ShimmerDriver/src/test/java/com/shimmerresearch/verisense/payloaddesign/API_00009_VerisenseSlowSensorGapWindow.java @@ -0,0 +1,634 @@ +package com.shimmerresearch.verisense.payloaddesign; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; + +import org.junit.Before; +import org.junit.Test; + +import com.shimmerresearch.driver.Configuration.COMMUNICATION_TYPE; +import com.shimmerresearch.driverUtilities.ShimmerVerDetails.HW_ID; +import com.shimmerresearch.sensors.AbstractSensor.SENSORS; +import com.shimmerresearch.verisense.VerisenseDevice; +import com.shimmerresearch.verisense.payloaddesign.DataBlockDetails.DATABLOCK_SENSOR_ID; +import com.shimmerresearch.verisense.sensors.SensorMLX90632; +import com.shimmerresearch.verisense.sensors.SensorVD6283; + +/** + * Unit tests for the DEV-979 slow-sensor timing refinement. These drive the REAL + * package-private methods on {@link PayloadContentsDetailsV8orAbove} - one call + * per synthetic payload, in sequence, with data blocks carrying end TICKS the + * way the metadata parse leaves them - and judge the resulting CSV split + * decisions through the real + * {@link UtilCsvSplitting#isDataBlockContinuous(SENSORS, DataSegmentDetails, DataBlockDetails)}. + * No binary test files, no hardware data and no reflection. + *

+ * The bug: the VD6283 is NOT duty-cycled. The firmware samples it on a plain + * repeated timer at one of {@code 0.5, 1, 2, 5, 10, 20} Hz + * (hal_slowSensorSampler.c {@code slowSensorRateMs[]}), defaulting to 1 Hz, and + * buffers 10 samples per block. That rate index lives in operational-config byte + * 75 and is NOT stored in the payload, so the parser fell back to the + * exposure-derived value - which only bounds the rate from ABOVE (10 Hz at the + * default 100 ms exposure). Each 10-sample block was therefore laid out over + * 0.9 s of the 10 s it really spans, and the 9.1 s remainder looked like a gap: + * 129 one-block CSVs. + *

+ * End-to-end coverage on the real recording is ASM_PC_00005_VerisenseFileParserPC + * Test_066; the DEV-927 skin-temp equivalent is Test_065. + */ +public class API_00009_VerisenseSlowSensorGapWindow { + + private static final int LIGHT_SAMPLES_PER_BLOCK = SensorVD6283.NUM_SAMPLES_PER_BLOCK; + private static final int SKIN_TEMP_SAMPLES_PER_BLOCK = SensorMLX90632.NUM_SAMPLES_PER_BLOCK; + + private static final double TICKS_PER_SECOND = AsmBinaryFileConstants.TICKS_PER_SECOND; + private static final long TICKS_PER_MINUTE = (long) AsmBinaryFileConstants.TICKS_PER_MINUTE; + + /** The DEV-979 recording: 1 Hz light, so a 10-sample block every 10 s. */ + private static final double LIGHT_1HZ_BLOCK_SPACING_S = 10; + /** Skin temp refresh code 6 = 32 Hz refresh -> 16 Hz medical output (DEV-927). */ + private static final int SKIN_TEMP_CONFIG_32HZ_REFRESH = 6<<1; + + @Before + public void clearSplittingState() { + UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor(); + } + + private VerisenseDevice setupGen2Device(int skinTempConfigByte) { + VerisenseDevice device = new VerisenseDevice(COMMUNICATION_TYPE.SD); + + byte[] configBytes = new byte[32]; + configBytes[0] = (byte) 0x10; // extended-config flag + configBytes[2] = 2; // FW major + configBytes[4] = 9; // FW internal LSB (v2.00.009) + configBytes[6] = (byte) 0xFF; // reset reason + configBytes[11] = HW_ID.VERISENSE_PULSE_PLUS; // SR68 + configBytes[12] = 9; // SR68-9 (second generation) + configBytes[25] = (byte) (0x02 | (1<<3) | (1<<4)); // GEN_CFG_3: LED + VD6283 + MLX90632 + configBytes[28] = (byte) skinTempConfigByte; // SKIN_TEMP_CONFIG + device.configBytesParse(configBytes, COMMUNICATION_TYPE.SD); + + device.getOrCreateListOfSensorClassKeysForDataBlockId(DATABLOCK_SENSOR_ID.LIGHT); + device.getOrCreateListOfSensorClassKeysForDataBlockId(DATABLOCK_SENSOR_ID.SKIN_TEMP); + assertTrue("this fixture must exercise the v11+ (uC ticks) path", device.isPayloadDesignV11orAbove()); + return device; + } + + private VerisenseDevice setupGen2Device() { + return setupGen2Device(0); + } + + private static int samplesPerBlock(DATABLOCK_SENSOR_ID slowSensorId) { + return slowSensorId==DATABLOCK_SENSOR_ID.LIGHT? LIGHT_SAMPLES_PER_BLOCK:SKIN_TEMP_SAMPLES_PER_BLOCK; + } + + private static SENSORS sensorClassKeyOf(DATABLOCK_SENSOR_ID slowSensorId) { + return slowSensorId==DATABLOCK_SENSOR_ID.LIGHT? SENSORS.VD6283:SENSORS.MLX90632; + } + + /** A block as the metadata parse leaves it: sized, timed with the header estimate, end TICKS set. */ + private DataBlockDetails newBlock(VerisenseDevice device, DATABLOCK_SENSOR_ID slowSensorId, long endTicks) { + int bytesPerSample = slowSensorId==DATABLOCK_SENSOR_ID.LIGHT? SensorVD6283.BYTES_PER_SAMPLE:SensorMLX90632.BYTES_PER_SAMPLE; + DataBlockDetails dataBlockDetails = new DataBlockDetails(slowSensorId, 0, 0, + device.getOrCreateListOfSensorClassKeysForDataBlockId(slowSensorId), 0, 0); + dataBlockDetails.setMetadata(samplesPerBlock(slowSensorId)*bytesPerSample, bytesPerSample, + device.getSamplingRateForSensor(sensorClassKeyOf(slowSensorId))); + // v11+ stores microcontroller-clock ticks per block; the sub-minute counter + // is what the refinement differences. + dataBlockDetails.getTimeDetailsUcClock().setEndTimeTicks(endTicks%TICKS_PER_MINUTE); + // The absolute RWC ms both the continuity check and the cross-payload + // measurement work on. Kept in step with the sub-minute ticks the + // in-payload path differences. + dataBlockDetails.getTimeDetailsRwc().setEndTimeMs(endTicks/TICKS_PER_SECOND*1000); + return dataBlockDetails; + } + + private static long ticks(double seconds) { + return (long) Math.round(seconds*TICKS_PER_SECOND); + } + + /** + * Run one payload through the real refinement: build a + * PayloadContentsDetailsV8orAbove, give it the blocks, and call the method the + * parse flow calls. + * + * @return the payload's blocks, as the refinement left them + */ + private DataBlockDetails[] refinePayload(VerisenseDevice device, DATABLOCK_SENSOR_ID slowSensorId, DataBlockDetails... payloadBlocks) { + PayloadContentsDetailsV8orAbove payloadContentsDetails = new PayloadContentsDetailsV8orAbove(device); + payloadContentsDetails.listOfDataBlocksInOrder.addAll(Arrays.asList(payloadBlocks)); + // The two phases the parse flow runs, in its order: apply the running + // estimate to this payload's blocks before their timings are back-filled... + payloadContentsDetails.refineSlowSensorSamplingRateFromBlockTicks(slowSensorId, samplesPerBlock(slowSensorId)); + // ...then, once they are, measure this payload's boundaries in absolute ms. + payloadContentsDetails.observeSlowSensorBlockSpacing(slowSensorId, samplesPerBlock(slowSensorId)); + return payloadBlocks; + } + + /** One payload holding exactly one block of the sensor, at the given end ticks. */ + private DataBlockDetails refineOneBlockPayload(VerisenseDevice device, DATABLOCK_SENSOR_ID slowSensorId, long endTicks) { + return refinePayload(device, slowSensorId, newBlock(device, slowSensorId, endTicks))[0]; + } + + private String continuityResult(DATABLOCK_SENSOR_ID slowSensorId, DataSegmentDetails previousSegment, DataBlockDetails next) { + return UtilCsvSplitting.isDataBlockContinuous(sensorClassKeyOf(slowSensorId), previousSegment, next); + } + + private DataSegmentDetails dataSegmentOf(DataBlockDetails... dataBlockDetails) { + DataSegmentDetails dataSegmentDetails = new DataSegmentDetails(); + for (DataBlockDetails block : dataBlockDetails) { + dataSegmentDetails.addDataBlock(block); + } + return dataSegmentDetails; + } + + /** + * Walk a stream of one-block payloads through the real refinement, asserting + * each boundary's split decision. + * + * @param spacingsS the spacing from each block to the next, in seconds + */ + private DataSegmentDetails walkStream(VerisenseDevice device, DATABLOCK_SENSOR_ID slowSensorId, double firstBlockEndS, double... spacingsS) { + double endS = firstBlockEndS; + DataBlockDetails previous = refineOneBlockPayload(device, slowSensorId, ticks(endS)); + DataSegmentDetails dataSegmentDetails = dataSegmentOf(previous); + for (int i = 0; i < spacingsS.length; i++) { + endS += spacingsS[i]; + DataBlockDetails next = refineOneBlockPayload(device, slowSensorId, ticks(endS)); + assertEquals("boundary " + i + " (spacing " + spacingsS[i] + " s) must be continuous", + "", continuityResult(slowSensorId, dataSegmentDetails, next)); + dataSegmentDetails.addDataBlock(next); + } + return dataSegmentDetails; + } + + private static double[] uniformSpacings(int count, double spacingS) { + double[] spacingsS = new double[count]; + Arrays.fill(spacingsS, spacingS); + return spacingsS; + } + + // ---------------------------------------------------------------- VD6283 + + /** + * The reported symptom, in the shape the firmware actually produces: 1 Hz + * light, a 10-sample block every 10 s. No boundary may split, so the whole + * stream lands in one CSV. + */ + @Test + public void test001_lightAt1HzDoesNotSplit() { + VerisenseDevice device = setupGen2Device(); + assertEquals("the header only yields the exposure-derived upper bound", + 10.0, device.getSamplingRateForSensor(SENSORS.VD6283), 1e-9); + + DataSegmentDetails dataSegmentDetails = walkStream(device, DATABLOCK_SENSOR_ID.LIGHT, 100, uniformSpacings(40, LIGHT_1HZ_BLOCK_SPACING_S)); + assertEquals(41, dataSegmentDetails.getDataBlockCount()); + } + + /** + * The refinement recovers the true 1 s period from the block spacing and + * applies it, so the blocks become CONTIGUOUS: each one's 10 samples span the + * 9 s from its first to its last, not the 0.9 s the exposure-derived estimate + * implied, and the next block starts one period after the previous one ends. + */ + @Test + public void test002_refinedPeriodIsAppliedAndMakesBlocksContiguous() { + VerisenseDevice device = setupGen2Device(); + + DataBlockDetails first = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(100)); + // The first block of a CSV set has nothing to measure against, so it keeps + // the header estimate - the documented residual. + assertEquals(10.0, first.getSamplingRate(), 1e-9); + assertEquals(0.1, first.getTimestampDiffInS(), 1e-9); + + // The second block completes the first boundary, but its own rate was applied + // before that boundary existed, so it keeps the estimate too - the residual + // is the first TWO blocks of a set. + DataBlockDetails second = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(100+LIGHT_1HZ_BLOCK_SPACING_S)); + assertEquals(10.0, second.getSamplingRate(), 1e-9); + + // From the third on, the measured 1 s period is applied + DataBlockDetails third = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(100+(2*LIGHT_1HZ_BLOCK_SPACING_S))); + assertEquals("10 samples over 10 s = 1 Hz", 1.0, third.getSamplingRate(), 1e-6); + assertEquals(1.0, third.getTimestampDiffInS(), 1e-6); + + // Timed from the end tick, the block's samples now span 9 x 1 s... + third.setUcClockEndTimeMinutesAndCalculateTimings(0); + double blockSpanMs = third.getTimeDetailsUcClock().getEndTimeMs()-third.getTimeDetailsUcClock().getStartTimeMs(); + assertEquals(9000, blockSpanMs, 1); + } + + /** + * A light configuration where the exposure-derived estimate happens to equal + * the truth (10 Hz: 10 samples 100 ms apart, a block every second) is refined + * to the same value, so nothing about it changes. + */ + @Test + public void test003_lightWhereTheEstimateEqualsTheTruthIsUnchanged() { + VerisenseDevice device = setupGen2Device(); + + walkStream(device, DATABLOCK_SENSOR_ID.LIGHT, 100, uniformSpacings(20, 1.0)); + + DataBlockDetails latest = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(121)); + assertEquals(10.0, latest.getSamplingRate(), 1e-6); + } + + /** + * A boundary that crosses a minute must still measure 1 Hz. The per-block + * stored time is a sub-minute counter, so the cross-payload measurement works + * on absolute real-world-clock ms instead and the crossing is a non-event - + * this pins that, since differencing the wrapped ticks would need a rebase. + */ + @Test + public void test004_minuteCrossingBoundaryIsMeasuredCorrectly() { + VerisenseDevice device = setupGen2Device(); + + // 45 s -> 55 s -> 65 s: the last block's sub-minute tick value is SMALLER + DataBlockDetails first = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(45)); + refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(55)); + DataBlockDetails third = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(65)); + assertTrue("the fixture must actually wrap the tick counter", + third.getTimeDetailsUcClock().getEndTimeTicks() 2 Hz. + double afterGapS = 100+(10*LIGHT_1HZ_BLOCK_SPACING_S)+65; + DataBlockDetails afterGap = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(afterGapS)); + + assertEquals("an over-a-minute gap must record NO observation", + observationsBefore, UtilCsvSplitting.getSlowSensorObservationCount(DATABLOCK_SENSOR_ID.LIGHT)); + assertEquals("the block keeps the prior median, not the aliased 2 Hz", + 1.0, afterGap.getSamplingRate(), 1e-6); + assertFalse("and the boundary is still reported as a split", + continuityResult(DATABLOCK_SENSOR_ID.LIGHT, dataSegmentDetails, afterGap).isEmpty()); + } + + /** A genuinely dropped block doubles the spacing and must still split. */ + @Test + public void test005_droppedLightBlockSplitsOnceTheHistoryExists() { + VerisenseDevice device = setupGen2Device(); + + DataSegmentDetails dataSegmentDetails = walkStream(device, DATABLOCK_SENSOR_ID.LIGHT, 100, uniformSpacings(20, LIGHT_1HZ_BLOCK_SPACING_S)); + + double afterDropoutS = 100+(20*LIGHT_1HZ_BLOCK_SPACING_S)+(LIGHT_1HZ_BLOCK_SPACING_S*2); + DataBlockDetails afterDropout = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(afterDropoutS)); + + assertFalse("a dropped light block must split the CSV", + continuityResult(DATABLOCK_SENSOR_ID.LIGHT, dataSegmentDetails, afterDropout).isEmpty()); + } + + /** + * On the FIRST boundary of a CSV set the window is a-priori, bounded by the + * firmware's rate table, so the slowest rate the hardware offers (0.5 Hz, a + * 10-sample block every 20 s) must not split. + */ + @Test + public void test006_firstBoundaryAtTheSlowestFirmwareRateDoesNotSplit() { + VerisenseDevice device = setupGen2Device(); + + double slowestSpacingS = LIGHT_SAMPLES_PER_BLOCK/SensorVD6283.MIN_SAMPLE_RATE_HZ; // 20 s + DataBlockDetails first = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(10)); + DataSegmentDetails dataSegmentDetails = dataSegmentOf(first); + DataBlockDetails second = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(10+slowestSpacingS)); + + assertEquals("0.5 Hz is a legitimate firmware rate and must not split", + "", continuityResult(DATABLOCK_SENSOR_ID.LIGHT, dataSegmentDetails, second)); + } + + /** + * ...but a gap beyond anything the hardware could produce must still split + * there. A 10-sample block 60 s after the previous one is 0.167 Hz, below the + * slowest firmware rate even after the standard ratio. + */ + @Test + public void test007_firstBoundaryWithA60SecondGapSplits() { + VerisenseDevice device = setupGen2Device(); + + DataBlockDetails first = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(10)); + DataSegmentDetails dataSegmentDetails = dataSegmentOf(first); + DataBlockDetails second = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(70)); + + assertFalse("a 60 s gap must split even on the first boundary of a set", + continuityResult(DATABLOCK_SENSOR_ID.LIGHT, dataSegmentDetails, second).isEmpty()); + } + + /** + * A failed I2C read makes ONE block take an extra sample period to fill + * without losing a sample slot (hal_slowSensorSampler.c only increments the + * count on a successful read), so that boundary measures 11 s. It must stay + * continuous and - because the median is applied, not the raw delta - must not + * stretch that block's or its neighbours' sample spacing. + */ + @Test + public void test008_i2cDroppedSampleStaysContinuousAndDoesNotStretchTheBlocks() { + VerisenseDevice device = setupGen2Device(); + + double[] spacingsS = uniformSpacings(20, LIGHT_1HZ_BLOCK_SPACING_S); + spacingsS[10] = LIGHT_1HZ_BLOCK_SPACING_S+1; // one sample dropped + DataSegmentDetails dataSegmentDetails = walkStream(device, DATABLOCK_SENSOR_ID.LIGHT, 100, spacingsS); + + for (DataBlockDetails dataBlockDetails : dataSegmentDetails.getListOfDataBlocks()) { + if(dataBlockDetails.getSamplingRate()!=10.0) { // skip the estimate-timed first block + assertEquals("no block may be stretched by the dropped sample", + 1.0, dataBlockDetails.getSamplingRate(), 1e-6); + } + } + } + + /** + * The plausibility filter refuses any period no configuration could have + * produced, so neither the running estimate nor a block's timing can be built + * from one. (Tick aliasing is handled at the source instead - see + * test019_aliasedOverMinuteGapIsNotLearnedFromOrApplied.) + */ + @Test + public void test009_aliasedGapIsNeitherRecordedNorApplied() { + VerisenseDevice device = setupGen2Device(); + + // A 65 s spacing is beyond anything the hardware can produce + DataBlockDetails first = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(10)); + DataSegmentDetails dataSegmentDetails = dataSegmentOf(first); + DataBlockDetails second = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(75)); + assertFalse("the real 65 s gap must still be reported", + continuityResult(DATABLOCK_SENSOR_ID.LIGHT, dataSegmentDetails, second).isEmpty()); + + // An implausible period is refused outright + assertFalse(UtilCsvSplitting.isSlowSensorPeriodPlausible(device, DATABLOCK_SENSOR_ID.LIGHT, 10.0)); + assertFalse(UtilCsvSplitting.isSlowSensorPeriodPlausible(device, DATABLOCK_SENSOR_ID.LIGHT, 0.001)); + assertFalse(UtilCsvSplitting.isSlowSensorPeriodPlausible(device, DATABLOCK_SENSOR_ID.LIGHT, Double.NaN)); + assertFalse(UtilCsvSplitting.isSlowSensorPeriodPlausible(device, DATABLOCK_SENSOR_ID.LIGHT, -1.0)); + assertTrue(UtilCsvSplitting.isSlowSensorPeriodPlausible(device, DATABLOCK_SENSOR_ID.LIGHT, 1.0)); + assertTrue(UtilCsvSplitting.isSlowSensorPeriodPlausible(device, DATABLOCK_SENSOR_ID.LIGHT, 2.0)); + + // ...and an implausible median is never applied, so the block keeps the estimate + UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor(); + UtilCsvSplitting.recordAndGetSlowSensorPeriodS(device, DATABLOCK_SENSOR_ID.LIGHT, 10.0); + assertEquals(0, UtilCsvSplitting.getSlowSensorObservationCount(DATABLOCK_SENSOR_ID.LIGHT)); + } + + /** + * A block a midday/midnight transition cut in two must be measured as the ONE + * whole block it is. The halves are a fraction of a second apart and carry + * reduced sample counts, so measuring them would fabricate both a far too fast + * and a far too slow observation - and the continuity check never sees the + * halves either, it recombines them. + */ + @Test + public void test010_middayMidnightSplitPartsAreMeasuredAsOneWholeBlock() { + VerisenseDevice device = setupGen2Device(); + + refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(10)); + + // The next block straddles the transition and is cut in two + DataBlockDetails firstPart = newBlock(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(20)); + DataBlockDetails secondPart = firstPart.deepClone(); + int splitAtSampleIndex = LIGHT_SAMPLES_PER_BLOCK/2; + firstPart.splitAndEndBeforeSampleIndex(splitAtSampleIndex, firstPart.getEndTimeRwcMs()-500, + firstPart.getTimeDetailsUcClock().getEndTimeMs()); + secondPart.splitAndStartAtSampleIndex(splitAtSampleIndex, secondPart.getEndTimeRwcMs()-400, + secondPart.getTimeDetailsUcClock().getEndTimeMs()); + // The first part's end tick moves with its end time; the SECOND part keeps + // the original block's end tick, which is what the refinement measures on. + firstPart.getTimeDetailsUcClock().setEndTimeTicks(ticks(19.5)); + assertTrue(firstPart.isFirstPartOfSplitDataBlock() && secondPart.isSecondPartOfSplitDataBlock()); + assertEquals(splitAtSampleIndex, firstPart.getSampleCount()); + assertEquals(LIGHT_SAMPLES_PER_BLOCK-splitAtSampleIndex, secondPart.getSampleCount()); + assertEquals(ticks(20), secondPart.getTimeDetailsUcClock().getEndTimeTicks()); + + refinePayload(device, DATABLOCK_SENSOR_ID.LIGHT, firstPart, secondPart); + + // One whole 10-sample block 10 s after the previous one = 1 Hz. Had the + // halves been measured separately the 0.5 s gap between them would have + // produced a wildly fast observation and a 5-sample slow one instead. + assertEquals(1, UtilCsvSplitting.getSlowSensorObservationCount(DATABLOCK_SENSOR_ID.LIGHT)); + assertEquals(1.0, UtilCsvSplitting.recordAndGetSlowSensorPeriodS(device, DATABLOCK_SENSOR_ID.LIGHT, Double.NaN), 1e-6); + } + + /** + * A CSV set that OPENS on an anomalous boundary must recover: nothing is + * excluded at learn time, because the history is empty after every clear and a + * learn-time plausibility test against the history would let the first + * boundary define what counts as plausible. + */ + @Test + public void test011_recoversFromAnomalousFirstObservation() { + VerisenseDevice device = setupGen2Device(); + + double[] spacingsS = uniformSpacings(20, LIGHT_1HZ_BLOCK_SPACING_S); + spacingsS[0] = LIGHT_1HZ_BLOCK_SPACING_S*2; // opens on a dropped block + walkStream(device, DATABLOCK_SENSOR_ID.LIGHT, 100, spacingsS); + + DataBlockDetails latest = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(100+(21*LIGHT_1HZ_BLOCK_SPACING_S))); + assertEquals("the estimate must recover onto the healthy period", 1.0, latest.getSamplingRate(), 1e-6); + } + + /** + * An overlapping (impossibly fast) boundary must be reported and must not + * define the fast side of the window, or it would widen it past itself and + * stop being reported. + */ + @Test + public void test012_overlappingBoundarySplitsAndDoesNotDefineTheFastSide() { + VerisenseDevice device = setupGen2Device(); + + DataSegmentDetails dataSegmentDetails = walkStream(device, DATABLOCK_SENSOR_ID.LIGHT, 100, uniformSpacings(20, LIGHT_1HZ_BLOCK_SPACING_S)); + + // A forward clock correction shrinks the spacing to 100 ms -> 100 Hz apparent + DataBlockDetails overlapping = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(100+(20*LIGHT_1HZ_BLOCK_SPACING_S)+0.1)); + assertFalse("an overlapping boundary must split", + continuityResult(DATABLOCK_SENSOR_ID.LIGHT, dataSegmentDetails, overlapping).isEmpty()); + assertTrue("the artefact must not define the fast side", + UtilCsvSplitting.isSamplingRateOutsideOfLimits(UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.get(SENSORS.VD6283), 100.0)); + } + + /** Writing a CSV set out clears every piece of slow-sensor state. */ + @Test + public void test013_clearResetsTheStateSoTheNextSetStartsFresh() { + VerisenseDevice device = setupGen2Device(); + + walkStream(device, DATABLOCK_SENSOR_ID.LIGHT, 100, uniformSpacings(5, LIGHT_1HZ_BLOCK_SPACING_S)); + assertTrue(UtilCsvSplitting.getSlowSensorObservationCount(DATABLOCK_SENSOR_ID.LIGHT)>0); + + UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor(); + + assertEquals(0, UtilCsvSplitting.getSlowSensorObservationCount(DATABLOCK_SENSOR_ID.LIGHT)); + // The first block of the new set is timed with the header estimate again - + // proof that no stale end tick or period survived. + DataBlockDetails afterClear = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(1000)); + assertEquals(10.0, afterClear.getSamplingRate(), 1e-9); + } + + /** Nothing changes for fast sensors. */ + @Test + public void test014_fastSensorLimitsAreUntouched() { + VerisenseDevice device = setupGen2Device(); + + double[] fastSensorLimits = UtilCsvSplitting.calculateSamplingRateLimits(960); + UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.put(SENSORS.LSM6DSV, fastSensorLimits); + + walkStream(device, DATABLOCK_SENSOR_ID.LIGHT, 100, uniformSpacings(10, LIGHT_1HZ_BLOCK_SPACING_S)); + + assertTrue("the fast sensor's band must be the same array, unmodified", + fastSensorLimits==UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.get(SENSORS.LSM6DSV)); + assertEquals(960*UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER.LOWER, fastSensorLimits[0], 1e-9); + assertNull("only the sensors of this data block are touched", + UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.get(SENSORS.MLX90632)); + } + + // ------------------------------------------------------------- MLX90632 + + /** + * The cross-payload measurement rests on the block end time being a + * SUB-MINUTE tick counter, so it is only sound while the sensor's largest + * legitimate block span is under a minute. A 10-sample light block spans at + * most 20 s; a 16-sample skin-temp block at its slowest output spans over a + * minute and must be refused. + */ + @Test + public void test015_crossPayloadMeasurementIsRefusedWhenTheTickDeltaIsAmbiguous() { + assertTrue("a 10-sample light block spans at most 20 s", + UtilCsvSplitting.isSlowSensorSpanUnambiguousAcrossPayloads(DATABLOCK_SENSOR_ID.LIGHT, LIGHT_SAMPLES_PER_BLOCK)); + assertFalse("a 16-sample skin-temp block can span more than a minute", + UtilCsvSplitting.isSlowSensorSpanUnambiguousAcrossPayloads(DATABLOCK_SENSOR_ID.SKIN_TEMP, SKIN_TEMP_SAMPLES_PER_BLOCK)); + assertFalse("a fast sensor is not a slow sensor", + UtilCsvSplitting.isSlowSensorSpanUnambiguousAcrossPayloads(DATABLOCK_SENSOR_ID.LSM6DSV, 100)); + } + + /** + * The MLX90632 fails the cross-payload gate, and at the DEV-927 16 Hz + * configuration a 16-sample block spans ~1 s, so it takes the pre-DEV-979 + * per-payload path. A payload holding a SINGLE temp block must then be left + * completely alone - no cross-payload measurement, no rate change, no window + * put. This is what makes skin-temp byte-identity hold by construction (the + * DEV-927 reference CSVs cannot be reached from here). The slower 0.25 Hz + * configuration is routed elsewhere - see + * {@link #test020_skinTempAtSlowestRateSeedsTheWindowFromTheHeaderNotTheWrappedTickDelta()}. + */ + @Test + public void test016_skinTempSingleBlockPayloadsAreLeftAlone() { + VerisenseDevice device = setupGen2Device(SKIN_TEMP_CONFIG_32HZ_REFRESH); + assertEquals("DEV-927 configuration is 16 Hz output", 16.0, device.getSamplingRateForSensor(SENSORS.MLX90632), 1e-9); + + DataBlockDetails first = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, ticks(1)); + DataBlockDetails second = refineOneBlockPayload(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, ticks(2)); + + assertEquals("the header-derived rate must be left in place", 16.0, first.getSamplingRate(), 1e-9); + assertEquals(16.0, second.getSamplingRate(), 1e-9); + assertEquals("no cross-payload history may be accumulated", + 0, UtilCsvSplitting.getSlowSensorObservationCount(DATABLOCK_SENSOR_ID.SKIN_TEMP)); + assertNull("no window may be seeded from a single-block payload", + UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.get(SENSORS.MLX90632)); + assertTrue(UtilCsvSplitting.SLOW_SENSOR_LAST_BLOCK_END_TIME_RWC_MS.isEmpty()); + } + + /** + * The DEV-927 shapes the reviewer measured against master, pinned here so the + * per-payload path cannot drift: 16 samples per block, 16 Hz output, 1000 ms + * nominal spacing, a +12.5% slip then a catch-up. The expected values are + * MASTER's - upper-middle median over this payload's periods only. + */ + @Test + public void test017_skinTempPerPayloadPathMatchesMasterForTheDev927Shapes() { + VerisenseDevice device = setupGen2Device(SKIN_TEMP_CONFIG_32HZ_REFRESH); + + // Two blocks in the payload: one boundary, the slip (1.125 s / 16 samples) + DataBlockDetails[] twoBlockPayload = refinePayload(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, + newBlock(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, ticks(10)), + newBlock(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, ticks(11.125))); + double expectedTwoBlockRate = 1.0/(1.125/SKIN_TEMP_SAMPLES_PER_BLOCK); + assertEquals("master applies this payload's own median", expectedTwoBlockRate, twoBlockPayload[0].getSamplingRate(), 1e-6); + assertEquals(expectedTwoBlockRate, twoBlockPayload[1].getSamplingRate(), 1e-6); + assertEquals("still no cross-payload history", 0, UtilCsvSplitting.getSlowSensorObservationCount(DATABLOCK_SENSOR_ID.SKIN_TEMP)); + + // Three blocks: two boundaries, the slip then the catch-up. Master's + // size()/2 UPPER-middle median of {0.875/16, 1.125/16} is the LARGER period. + UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor(); + DataBlockDetails[] threeBlockPayload = refinePayload(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, + newBlock(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, ticks(10)), + newBlock(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, ticks(11.125)), + newBlock(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, ticks(12))); + double upperMiddlePeriodS = 1.125/SKIN_TEMP_SAMPLES_PER_BLOCK; + assertEquals("master takes the upper-middle median, not the mean of the two", + 1.0/upperMiddlePeriodS, threeBlockPayload[0].getSamplingRate(), 1e-6); + + // Master's window: gap side achievedRate/1.5, fast side (1/minPeriod)*1.1 + double[] samplingRateLimits = UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.get(SENSORS.MLX90632); + assertEquals((1.0/upperMiddlePeriodS)/UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO, + samplingRateLimits[0], 1e-6); + assertEquals((1.0/(0.875/SKIN_TEMP_SAMPLES_PER_BLOCK))*UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER.UPPER, + samplingRateLimits[1], 1e-6); + } + + /** + * The MLX90632 at its slowest configuration (refresh code 0 = 0.5 Hz refresh + * -> 0.25 Hz medical output, a 16-sample block spanning ~64 s). Two such + * blocks CAN land in one payload, and the pre-DEV-979 per-payload path would + * difference their SUB-MINUTE end-tick counters: a real 64 s gap re-bases by + * one minute to ~4 s, i.e. an apparent ~4 Hz, and gets written unconditionally + * into the window (fast side ~4.8 Hz) - after which every real 0.25 Hz + * boundary reads as a time-gap and the CSV splits per block. + *

+ * The refresh code is in the header, so this configuration is instead routed + * to a window seeded straight from the header rate, and no tick differencing + * is done. + */ + @Test + public void test020_skinTempAtSlowestRateSeedsTheWindowFromTheHeaderNotTheWrappedTickDelta() { + VerisenseDevice device = setupGen2Device(0); // refresh code 0 -> 0.5 Hz refresh -> 0.25 Hz output + assertEquals("slowest configuration is 0.25 Hz output", 0.25, + device.getSamplingRateForSensor(SENSORS.MLX90632), 1e-9); + + // Two 16-sample blocks 64 s apart in one payload - the shape that aliased. + DataBlockDetails[] payload = refinePayload(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, + newBlock(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, ticks(10)), + newBlock(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, ticks(74))); + + assertEquals("the block keeps its header-derived rate, not the wrapped ~4 Hz", + 0.25, payload[0].getSamplingRate(), 1e-9); + assertEquals(0.25, payload[1].getSamplingRate(), 1e-9); + assertEquals("no per-payload tick differencing, no cross-payload history", + 0, UtilCsvSplitting.getSlowSensorObservationCount(DATABLOCK_SENSOR_ID.SKIN_TEMP)); + + double[] samplingRateLimits = UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.get(SENSORS.MLX90632); + assertNotNull("the gap window is seeded from the header rate", samplingRateLimits); + assertTrue("the window sits around 0.25 Hz, not the wrapped ~4 Hz (fast side was ~4.8 with the bug)", + samplingRateLimits[1] < 1.0); + + // A genuine 64 s boundary between 0.25 Hz blocks must NOT split. + UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor(); + walkStream(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, 10, 64, 64, 64); + } + + /** The median helper averages the two middle values for an even-sized input. */ + @Test + public void test018_medianIsTheMeanOfTheTwoMiddleValues() { + assertTrue(Double.isNaN(UtilCsvSplitting.calculateMedian(Arrays.asList()))); + assertEquals(2.0, UtilCsvSplitting.calculateMedian(Arrays.asList(1.0, 2.0, 5.0)), 1e-9); + assertEquals(1.5, UtilCsvSplitting.calculateMedian(Arrays.asList(1.0, 2.0)), 1e-9); + // The upper-middle median the legacy path uses would have returned 4.0 here + assertEquals(1.5, UtilCsvSplitting.calculateMedian(Arrays.asList(1.0, 1.0, 2.0, 4.0)), 1e-9); + } +}