diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java index c7db17b9..9943a848 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.ListIterator; import java.util.Map.Entry; @@ -324,13 +325,6 @@ private boolean isParserAtEndOfBuffer(int bufferLength, int currentByteIndex) { * {@code inter-block ticks / samples-per-block} is the exact per-sample period. * With fewer than two blocks in the payload the header-derived estimate the * blocks were created with is left in place. - *

- * The measurements are accumulated per sensor across the payloads of the - * current parse run (see - * {@link UtilCsvSplitting#refineSlowSensorSamplingRateLimits(SENSORS, java.util.List)}), - * and it is the median over that history - not this payload's handful of - * inter-block gaps - that is applied to the blocks and used to (re)derive the - * CSV gap-splitting window. */ private void refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID slowSensorId) { List slowSensorBlocks = new ArrayList(); @@ -367,61 +361,50 @@ private void refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID slow if(perSamplePeriodsS.isEmpty()) { return; } - - // Feed this payload's measurements into the sensor's running history and take - // the median over EVERYTHING measured so far in this parse run, then seed the - // CSV gap-splitting window from it. - // - // Why accumulate rather than re-derive the window from this payload alone: - // the checker (UtilCsvSplitting.isSamplingRateOutsideOfLimits) computes - // exactly 1/period for each block boundary, i.e. the very quantities measured - // here. A payload only carries 2-3 slow-sensor blocks, so a window re-derived - // from just this payload would absorb a dropped block's 2x spacing into its - // own median and never flag it - while the healthy boundary back to the - // previous payload got flagged instead. Against a history spanning hundreds - // of payloads a single 2x outlier barely moves the median, so the real gap - // stays outside the window. - // - // Why the window is needed at all: the header-derived estimate the blocks were - // created with can sit within ~1% of a +/-10% band edge (VD6283: 10 Hz - // estimated vs ~9.09 Hz achieved), and the slow sensors' cadence is inherently - // jittery - the light's is bimodal (exposure vs exposure + dead time: ~100 vs - // ~110 ms at the default exposure) and the MLX90632's conversions can slip by - // several refresh periods and then catch up (observed +12.5% block spacing - // with no samples lost - DEV-927 validation data). - double achievedRateHz = Double.NaN; - for(SENSORS sensorClassKey:verisenseDevice.getOrCreateListOfSensorClassKeysForDataBlockId(slowSensorId)) { - if(sensorClassKey!=SENSORS.CLOCK) { - double medianRateHz = UtilCsvSplitting.refineSlowSensorSamplingRateLimits(sensorClassKey, perSamplePeriodsS); - if(Double.isNaN(achievedRateHz)) { - // All of a data block's sensor class keys are fed the same - // measurements, so they all return the same median - just keep the - // first one for the block sampling rate below. - achievedRateHz = medianRateHz; - } - } - } - if(Double.isNaN(achievedRateHz)) { - // No sensor class key was accumulated against (nothing but CLOCK mapped to - // this data block id), so fall back to this payload's own median. - double medianPeriodS = UtilCsvSplitting.calculateMedian(perSamplePeriodsS); - achievedRateHz = medianPeriodS>0? 1.0/medianPeriodS:Double.NaN; - } - if(!(achievedRateHz>0)) { + // Median so a dropped block (a 2x gap) can't skew the period. + Collections.sort(perSamplePeriodsS); + double medianPeriodS = perSamplePeriodsS.get(perSamplePeriodsS.size()/2); + if(!(medianPeriodS>0)) { return; } - // The blocks are given the same accumulated median the gap window is built - // from, rather than this payload's own possibly-skewed median: the rate is - // what the block start times (and hence the CSV timestamps) are back-filled - // with, so a payload that happens to contain a dropped block would otherwise - // stretch its own samples' spacing by the very artefact the window is meant to - // report. Keeping both on one estimate also stops the timestamps and the - // continuity check disagreeing about what the achieved cadence is. + double achievedRateHz = 1.0/medianPeriodS; for(DataBlockDetails dataBlockDetails:slowSensorBlocks) { dataBlockDetails.setSamplingRate(achievedRateHz); dataBlockDetails.calculateTimestampDiffInS(); } + + // Seed the CSV gap-splitting window from the OBSERVED cadence rather than a + // single-rate +/-10% band. The header-derived estimate can sit within ~1% of + // the band edge (VD6283: 10 Hz estimated vs ~9.09 Hz achieved), and the slow + // sensors' cadence is inherently jittery: the light's is bimodal (exposure vs + // exposure + dead time: ~100 vs ~110 ms at the default exposure) and the + // MLX90632's conversions can slip by several refresh periods and then catch + // up (observed +12.5% block spacing with no samples lost - DEV-927 + // validation data). A single payload carries only 2-3 slow-sensor blocks, + // i.e. one or two inter-block gaps - no spread information - so the gap + // side of the window cannot rely on observed spread at all: it is set to + // tolerate anything up to SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO x the + // achieved median spacing, which keeps healthy jitter continuous while a + // genuinely dropped block (2x spacing) still splits. The fast side keeps + // the observed-minimum-period basis with the standard tolerance. + // The put is deliberately UNCONDITIONAL: the limits map is global across + // payloads, and a payload with fewer than two blocks of this sensor (early + // return above - e.g. the very first payload of a recording) leaves + // populateExpectedPayloadTsDiffLimitMapIfNeeded to seed a configured-rate + // +/-10% band first. A containsKey guard here would then lock that too-tight + // estimate in for the whole file (observed: 25-min DEV-927 skin-temp + // recording fragmented into 7 CSVs); the measured window must win as soon as + // it exists, and re-measuring on every payload keeps it tracking the sensor. + double minPeriodS = perSamplePeriodsS.get(0); + double[] samplingRateLimits = new double[] { + achievedRateHz/UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO, + (1.0/minPeriodS)*UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER.UPPER}; + for(SENSORS sensorClassKey:verisenseDevice.getOrCreateListOfSensorClassKeysForDataBlockId(slowSensorId)) { + if(sensorClassKey!=SENSORS.CLOCK) { + UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.put(sensorClassKey, samplingRateLimits); + } + } } private void backfillDataBlockRwcTimestamps() { diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java index 1eba6a51..f6c23ee0 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/UtilCsvSplitting.java @@ -1,7 +1,5 @@ package com.shimmerresearch.verisense.payloaddesign; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -13,61 +11,26 @@ public class UtilCsvSplitting { - public static class FILE_GAP_TOLERANCE_MULTIPLIER { + public class FILE_GAP_TOLERANCE_MULTIPLIER { // +/- 10% public static final double UPPER = 1.1; public static final double LOWER = 0.9; /** * Slow sensors only (VD6283 light / MLX90632 skin temp): the largest * inter-block gap, as a multiple of the achieved median block spacing, that - * is still treated as continuous. It sets the SLOW (gap) side of the - * sampling-rate window only - the fast side stays on the standard UPPER - * (+10%) tolerance, see {@link - * UtilCsvSplitting#calculateSlowSensorSamplingRateLimits(double)}. - *

- * The median it is applied to is accumulated across every payload parsed so - * far in the current parse run (see - * {@link UtilCsvSplitting#refineSlowSensorSamplingRateLimits(SENSORS, List)}), - * not re-derived from the handful of inter-block gaps in the payload - * currently being judged - a payload only carries 2-3 slow-sensor blocks, so - * a per-payload estimate would absorb a dropped block into its own window and - * never report it. Against the accumulated median a single 2x outlier barely - * moves the centre, so the gap stays outside the window. - *

- * The band still has to be wide because the slow sensors' cadence is - * inherently jittery even when no samples are lost: the light's is bimodal - * (exposure vs exposure + dead time) and the MLX90632's conversions can slip - * by several refresh periods and then catch up (observed up to +12.5% block - * spacing on the DEV-927 validation recording with no samples lost), which - * routinely violates the standard LOWER (-10%) band. A genuinely dropped - * block doubles the spacing (2x), so 1.5x sits comfortably between healthy - * jitter and a real gap. + * is still treated as continuous. The MLX90632's conversions can slip by + * several refresh periods and then catch up (observed up to +12.5% block + * spacing on the DEV-927 validation recording with no samples lost), and the + * window is seeded from the first payload that carries >= 2 blocks - often a + * single inter-block gap, i.e. no spread information - so the standard + * LOWER (-10%) band is routinely violated by healthy data. A genuinely + * dropped block doubles the spacing (2x), so 1.5x keeps comfortable margin + * on both sides. */ public static final double SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO = 1.5; } - - /** - * The maximum number of slow-sensor per-sample periods kept per sensor in - * {@link #SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR}. Once full, the oldest - * measurements are dropped so that the median follows any genuine long-term - * drift in the sensor's cadence while staying deep enough (hundreds of - * payloads' worth of inter-block gaps) that individual dropped blocks cannot - * shift it. - */ - protected static final int SLOW_SENSOR_PERIOD_HISTORY_MAX = 1024; - - protected static HashMap SAMPLING_RATE_LIMITS_PER_SENSOR = new HashMap(); - - /** - * Slow-sensor (VD6283 light / MLX90632 skin temp) per-sample periods, in - * seconds, as measured from the inter-block tick spacing of every payload - * parsed so far in the current parse run. Shares its lifecycle with - * {@link #SAMPLING_RATE_LIMITS_PER_SENSOR}: both are cleared together by - * {@link #clearMapOfSamplingRateLimitsPerSensor()}, which the file parser calls - * on each CSV-set boundary so that measurements never leak from one recording - * into the next. - */ - protected static HashMap> SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR = new HashMap>(); + + protected static HashMap SAMPLING_RATE_LIMITS_PER_SENSOR = new HashMap(); public static boolean isTsDifferenceOutsideOfLimits(double expectedPayloadTsDiffLimits[], double unixTimeInMs_1, double unixTimeInMs_2) { double differenceInMillisec = Math.abs(unixTimeInMs_1 - unixTimeInMs_2); @@ -136,116 +99,8 @@ public static double[] calculateSamplingRateLimits(double configuredSamplingRate return new double[] {configuredSamplingRate*FILE_GAP_TOLERANCE_MULTIPLIER.LOWER, configuredSamplingRate*FILE_GAP_TOLERANCE_MULTIPLIER.UPPER}; } - /** - * Slow-sensor (VD6283 light / MLX90632 skin temp) window either side of the - * achieved median rate. Both sides are derived from the SAME robust median so - * that neither edge can be dragged around by a single extreme inter-block - * spacing: the slow (gap) side tolerates up to - * {@link FILE_GAP_TOLERANCE_MULTIPLIER#SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO} - * times the median spacing, the fast side the standard - * {@link FILE_GAP_TOLERANCE_MULTIPLIER#UPPER} tolerance. - * - * @param medianRateHz the achieved median sampling rate, in Hz - * @return {min, max} sampling rate, in Hz, still treated as continuous - */ - public static double[] calculateSlowSensorSamplingRateLimits(double medianRateHz) { - return new double[] { - medianRateHz/FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO, - medianRateHz*FILE_GAP_TOLERANCE_MULTIPLIER.UPPER}; - } - - /** - * Median of the supplied values. Unlike a bare {@code get(size/2)} this - * averages the two middle values for an even-sized input, and it sorts a copy - * so the caller's list ordering is left alone. - * - * @param values the values to take the median of - * @return the median, or {@link Double#NaN} if there are no values - */ - public static double calculateMedian(List values) { - if(values==null || values.isEmpty()) { - return Double.NaN; - } - List sortedValues = new ArrayList(values); - Collections.sort(sortedValues); - int size = sortedValues.size(); - if(size%2==0) { - return (sortedValues.get((size/2)-1) + sortedValues.get(size/2))/2.0; - } - return sortedValues.get(size/2); - } - - /** - * Add the per-sample periods measured in the payload just parsed to this - * sensor's running history and return the median over EVERYTHING accumulated so - * far in the current parse run (not just the latest payload's values). - * - * @param sensorClassKey the sensor the periods were measured for - * @param newlyObservedPeriodsS the per-sample periods, in seconds, measured in - * the payload just parsed (may be empty/null to just read the - * current median back) - * @return the accumulated median per-sample period, in seconds, or - * {@link Double#NaN} if nothing has been measured for this sensor yet - */ - public static double accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSORS sensorClassKey, List newlyObservedPeriodsS) { - List accumulatedPeriodsS = SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR.get(sensorClassKey); - if(accumulatedPeriodsS==null) { - accumulatedPeriodsS = new ArrayList(); - SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR.put(sensorClassKey, accumulatedPeriodsS); - } - if(newlyObservedPeriodsS!=null) { - for(Double periodS:newlyObservedPeriodsS) { - if(periodS!=null && periodS>0) { - accumulatedPeriodsS.add(periodS); - } - } - } - // Bounded history: drop the oldest measurements rather than growing without - // limit over a multi-day recording. - int excess = accumulatedPeriodsS.size()-SLOW_SENSOR_PERIOD_HISTORY_MAX; - if(excess>0) { - accumulatedPeriodsS.subList(0, excess).clear(); - } - return calculateMedian(accumulatedPeriodsS); - } - - /** - * Accumulate the slow-sensor per-sample periods measured in the payload just - * parsed and (re)apply the resulting CSV gap-splitting window for that sensor. - *

- * The put into {@link #SAMPLING_RATE_LIMITS_PER_SENSOR} is deliberately - * UNCONDITIONAL. A payload that carries fewer than two blocks of this sensor - * (e.g. the very first payload of a recording) leaves - * {@link #populateExpectedPayloadTsDiffLimitMapIfNeeded(VerisenseDevice, HashMap)} - * to seed a configured-rate +/-10% band first; the header-derived rates for the - * slow sensors are only estimates (the light rate isn't stored at all), so that - * band can be far too tight (observed: a 25-min DEV-927 skin-temp recording - * fragmented into 7 CSVs). A containsKey guard here would lock that estimate in - * for the whole file, so the measured window must win as soon as it exists. - * - * @param sensorClassKey the sensor the periods were measured for - * @param newlyObservedPeriodsS the per-sample periods, in seconds, measured in - * the payload just parsed - * @return the accumulated median sampling rate, in Hz, or {@link Double#NaN} if - * nothing has been measured for this sensor yet (in which case the - * limits map is left untouched) - */ - public static double refineSlowSensorSamplingRateLimits(SENSORS sensorClassKey, List newlyObservedPeriodsS) { - double medianPeriodS = accumulateSlowSensorPeriodsAndGetMedianPeriodS(sensorClassKey, newlyObservedPeriodsS); - if(!(medianPeriodS>0)) { - return Double.NaN; - } - double medianRateHz = 1.0/medianPeriodS; - SAMPLING_RATE_LIMITS_PER_SENSOR.put(sensorClassKey, calculateSlowSensorSamplingRateLimits(medianRateHz)); - return medianRateHz; - } - public static void clearMapOfSamplingRateLimitsPerSensor() { SAMPLING_RATE_LIMITS_PER_SENSOR.clear(); - // Same lifecycle as the limits map itself - the accumulated slow-sensor - // measurements that the limits are derived from must not survive a CSV-set - // boundary either. - SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR.clear(); } public static String isDataBlockContinuous(SENSORS sensorClassKey, DataSegmentDetails dataSegmentDetailsPrevious, DataBlockDetails nextDataBlockDetails) { diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/verisense/payloaddesign/API_00009_UtilCsvSplittingSlowSensorGapWindow.java b/ShimmerDriver/src/test/java/com/shimmerresearch/verisense/payloaddesign/API_00009_UtilCsvSplittingSlowSensorGapWindow.java deleted file mode 100644 index 7a8f5dd1..00000000 --- a/ShimmerDriver/src/test/java/com/shimmerresearch/verisense/payloaddesign/API_00009_UtilCsvSplittingSlowSensorGapWindow.java +++ /dev/null @@ -1,275 +0,0 @@ -package com.shimmerresearch.verisense.payloaddesign; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; - -import org.junit.Before; -import org.junit.Test; - -import com.shimmerresearch.driver.Configuration.COMMUNICATION_TYPE; -import com.shimmerresearch.sensors.AbstractSensor.SENSORS; -import com.shimmerresearch.verisense.VerisenseDevice; -import com.shimmerresearch.verisense.payloaddesign.DataBlockDetails.DATABLOCK_SENSOR_ID; -import com.shimmerresearch.verisense.payloaddesign.UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER; - -/** - * Unit tests for the slow-sensor (VD6283 light / MLX90632 skin temp) CSV - * gap-splitting window in {@link UtilCsvSplitting} - the accumulated-median - * estimate that - * {@code PayloadContentsDetailsV8orAbove.refineSlowSensorSamplingRateFromBlockTicks} - * feeds on every payload. - *

- * The tests drive the window through per-sample periods, in seconds, exactly as - * the payload parser measures them ({@code inter-block ticks / samples-per-block}), - * so no binary test files or hardware recordings are needed. The boundary rate - * the CSV splitter then judges is simply {@code 1/period} - see - * {@link UtilCsvSplitting#isSamplingRateOutsideOfLimits(double[], DataBlockDetails, DataBlockDetails, SENSORS)}, - * which computes samples/second between two consecutive block end times. - *

- * End-to-end coverage against real recordings lives in - * ASM_PC_00005_VerisenseFileParserPC (ASM_PC repository). - */ -public class API_00009_UtilCsvSplittingSlowSensorGapWindow { - - /** ~9.09 Hz - the achieved VD6283 cadence at the default exposure. */ - private static final double NOMINAL_PERIOD_S = 0.11; - private static final double NOMINAL_RATE_HZ = 1.0/NOMINAL_PERIOD_S; - /** Number of healthy payloads used to build up a history before the payload under test. */ - private static final int HEALTHY_PAYLOAD_COUNT = 20; - /** Slow-sensor blocks per payload is 2-3 in the field, i.e. 1-2 inter-block gaps. */ - private static final int GAPS_PER_PAYLOAD = 2; - - private static final SENSORS SENSOR_UNDER_TEST = SENSORS.VD6283; - - private static final double DELTA = 1e-9; - - @Before - public void resetStaticState() { - // The limits map and the period history are process-wide statics, cleared by - // the file parser at each CSV-set boundary - do the same between tests so - // that they cannot leak into one another. - UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor(); - } - - // ---------------------------------------------------------------- helpers - - /** Feed one payload's worth of measurements in, as the parser does per payload. */ - private double feedPayload(double... perSamplePeriodsS) { - List periodsS = new ArrayList(); - for(double periodS:perSamplePeriodsS) { - periodsS.add(periodS); - } - return UtilCsvSplitting.refineSlowSensorSamplingRateLimits(SENSOR_UNDER_TEST, periodsS); - } - - /** Build up a history of healthy payloads at the nominal cadence. */ - private void feedHealthyHistory() { - for(int i=0;i callersList = new ArrayList(Arrays.asList(4.0, 1.0, 3.0, 2.0)); - UtilCsvSplitting.calculateMedian(callersList); - assertEquals(Arrays.asList(4.0, 1.0, 3.0, 2.0), callersList); - // Nothing measured yet. - assertTrue(Double.isNaN(UtilCsvSplitting.calculateMedian(new ArrayList()))); - assertTrue(Double.isNaN(UtilCsvSplitting.calculateMedian(null))); - } - - @Test - public void testAccumulatedMedianAveragesTheTwoMiddleValuesForAnEvenCount() { - // Four measurements across two payloads -> the mean of the middle two. - feedPayload(0.10, 0.12); - double medianPeriodS = UtilCsvSplitting.accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSOR_UNDER_TEST, Arrays.asList(0.14, 0.16)); - assertEquals(0.13, medianPeriodS, DELTA); - } - - // ------------------------------------------------------- window geometry - - @Test - public void testBothLimitsAreDerivedFromTheSameMedian() { - double[] limits = UtilCsvSplitting.calculateSlowSensorSamplingRateLimits(NOMINAL_RATE_HZ); - assertEquals(NOMINAL_RATE_HZ/FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO, limits[0], DELTA); - assertEquals(NOMINAL_RATE_HZ*FILE_GAP_TOLERANCE_MULTIPLIER.UPPER, limits[1], DELTA); - - // A single fast outlier in the history must not push the fast side out with - // it - the old limits[1] used 1/minObservedPeriod, i.e. that extremum. - feedHealthyHistory(); - double fastOutlierPeriodS = NOMINAL_PERIOD_S/2.0; - feedPayload(NOMINAL_PERIOD_S, fastOutlierPeriodS); - assertArrayEquals(UtilCsvSplitting.calculateSlowSensorSamplingRateLimits(NOMINAL_RATE_HZ), currentLimits(), 1e-6); - assertTrue(UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(fastOutlierPeriodS))); - } - - // ------------------------------------------------------ (b) healthy jitter - - @Test - public void testHealthyJitterStaysInsideTheWindow() { - feedHealthyHistory(); - - // +12.5% block spacing with no samples lost - observed on the DEV-927 - // MLX90632 validation recording. This payload contributes to the estimate. - double jitteredPeriodS = NOMINAL_PERIOD_S*1.125; - feedPayload(NOMINAL_PERIOD_S, jitteredPeriodS); - - double[] limits = currentLimits(); - assertFalse("+12.5% block spacing must still be judged continuous", - UtilCsvSplitting.isSamplingRateOutsideOfLimits(limits, boundaryRateHz(jitteredPeriodS))); - assertFalse("the nominal cadence must obviously be judged continuous", - UtilCsvSplitting.isSamplingRateOutsideOfLimits(limits, boundaryRateHz(NOMINAL_PERIOD_S))); - // A single +12.5% sample barely moves the median off the nominal cadence. - assertEquals(NOMINAL_RATE_HZ, 1.0/UtilCsvSplitting.accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSOR_UNDER_TEST, null), 1e-6); - } - - @Test - public void testHealthyJitterStaysInsideTheWindowFromTheVeryFirstPayload() { - // No history at all yet: the first payload with >= 2 blocks is all there is, - // and the window must already be wide enough for the jitter it contains. - double jitteredPeriodS = NOMINAL_PERIOD_S*1.125; - feedPayload(NOMINAL_PERIOD_S, jitteredPeriodS); - assertFalse(UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(jitteredPeriodS))); - assertFalse(UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(NOMINAL_PERIOD_S))); - } - - // ------------------------------------------------------- (c) dropped block - - @Test - public void testDroppedBlockIsDetectedEvenThoughItsPayloadFedTheEstimate() { - feedHealthyHistory(); - - // A dropped block doubles the spacing. This payload is fed into the estimate - // BEFORE the boundary it contains is judged - exactly the self-referential - // case that a per-payload window could not detect. - double droppedBlockPeriodS = NOMINAL_PERIOD_S*2.0; - feedPayload(NOMINAL_PERIOD_S, droppedBlockPeriodS); - - assertTrue("a 2x inter-block gap must fall outside the window", - UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(droppedBlockPeriodS))); - // The healthy boundary in the same payload must NOT be flagged instead. - assertFalse(UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(NOMINAL_PERIOD_S))); - // One 2x outlier in ~40 measurements leaves the median where it was. - assertEquals(NOMINAL_RATE_HZ, 1.0/UtilCsvSplitting.accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSOR_UNDER_TEST, null), 1e-6); - } - - @Test - public void testPerPayloadWindowWouldHaveAbsorbedTheDroppedBlock() { - // Regression guard for the finding this change addresses: a window rebuilt - // from ONLY the payload being judged (2 blocks -> 1 or 2 gaps) swallows the - // dropped block into its own centre and reports nothing. - double droppedBlockPeriodS = NOMINAL_PERIOD_S*2.0; - double payloadLocalMedianPeriodS = UtilCsvSplitting.calculateMedian(Arrays.asList(NOMINAL_PERIOD_S, droppedBlockPeriodS)); - double[] payloadLocalLimits = UtilCsvSplitting.calculateSlowSensorSamplingRateLimits(1.0/payloadLocalMedianPeriodS); - assertFalse("baseline: a payload-local window does NOT see the dropped block", - UtilCsvSplitting.isSamplingRateOutsideOfLimits(payloadLocalLimits, boundaryRateHz(droppedBlockPeriodS))); - - // The accumulated window does. - feedHealthyHistory(); - feedPayload(NOMINAL_PERIOD_S, droppedBlockPeriodS); - assertTrue(UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(droppedBlockPeriodS))); - } - - // ------------------------------------------- (d) fallback seeding interplay - - @Test - public void testMeasuredWindowWinsOverAFallbackSeededBand() { - // populateExpectedPayloadTsDiffLimitMapIfNeeded seeds a configured-rate - // +/-10% band for any sensor not yet measured - e.g. after a first payload - // that carried fewer than two blocks of this sensor. - double configuredRateHz = 10.0; - UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.put(SENSOR_UNDER_TEST, UtilCsvSplitting.calculateSamplingRateLimits(configuredRateHz)); - - // The header-derived 10 Hz is an estimate; the achieved cadence is ~9.09 Hz, - // which the +/-10% band already calls a gap on every single boundary (this - // fragmented a 25-min DEV-927 recording into 7 CSVs). - assertTrue("baseline: the fallback band is too tight for the achieved cadence", - UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(NOMINAL_PERIOD_S*1.125))); - - // First measurement must take over immediately - no containsKey guard. - feedPayload(NOMINAL_PERIOD_S, NOMINAL_PERIOD_S); - assertArrayEquals(UtilCsvSplitting.calculateSlowSensorSamplingRateLimits(NOMINAL_RATE_HZ), currentLimits(), 1e-6); - assertFalse(UtilCsvSplitting.isSamplingRateOutsideOfLimits(currentLimits(), boundaryRateHz(NOMINAL_PERIOD_S*1.125))); - } - - @Test - public void testFallbackSeedingDoesNotClobberAnExistingMeasuredWindow() { - feedHealthyHistory(); - double[] measuredLimits = currentLimits().clone(); - - // populateExpectedPayloadTsDiffLimitMapIfNeeded runs after the refinement on - // every payload; its containsKey guard must leave the measurement alone. - HashMap> mapOfSensorIdsPerDataBlock = new HashMap>(); - mapOfSensorIdsPerDataBlock.put(DATABLOCK_SENSOR_ID.LIGHT, Arrays.asList(SENSOR_UNDER_TEST)); - UtilCsvSplitting.populateExpectedPayloadTsDiffLimitMapIfNeeded(new VerisenseDevice(COMMUNICATION_TYPE.SD), mapOfSensorIdsPerDataBlock); - - assertArrayEquals(measuredLimits, currentLimits(), DELTA); - } - - // -------------------------------------------------------- lifecycle contract - - @Test - public void testClearingTheLimitsMapAlsoClearsTheAccumulatedPeriods() { - feedHealthyHistory(); - assertEquals(NOMINAL_PERIOD_S, UtilCsvSplitting.accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSOR_UNDER_TEST, null), DELTA); - - // The file parser calls this on each CSV-set boundary: measurements from one - // recording must not survive into the next. - UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor(); - - assertTrue(UtilCsvSplitting.SLOW_SENSOR_OBSERVED_PERIODS_PER_SENSOR.isEmpty()); - assertTrue(Double.isNaN(UtilCsvSplitting.accumulateSlowSensorPeriodsAndGetMedianPeriodS(SENSOR_UNDER_TEST, null))); - assertTrue(currentLimits()==null || currentLimits().length==0); - } - - @Test - public void testPeriodHistoryIsBounded() { - int payloadsToOverflowHistory = (UtilCsvSplitting.SLOW_SENSOR_PERIOD_HISTORY_MAX/GAPS_PER_PAYLOAD)+10; - for(int i=0;i NaN and no limits written. - assertTrue(Double.isNaN(UtilCsvSplitting.refineSlowSensorSamplingRateLimits(SENSOR_UNDER_TEST, new ArrayList()))); - assertTrue(currentLimits()==null); - - // Zero/negative periods (a same-tick or out-of-order block pair) are dropped - // rather than dragging the median to zero. - assertEquals(NOMINAL_RATE_HZ, feedPayload(0.0, -1.0, NOMINAL_PERIOD_S), 1e-6); - } -}