+ * DIAGNOSTIC USE ONLY - no parsing decision keys on it. The rate field is + * self-describing: earlier firmware always wrote zero into those bits, and + * this firmware writes the EFFECTIVE index, which is never zero while light + * blocks exist. So a parser can simply ask whether the field is set. This + * constant exists to tell "old recording" apart from "new recording, but + * the firmware failed to populate the field", which is a bug worth + * reporting rather than silently tolerating. + *
+ * PLACEHOLDER: confirm against the firmware release that ships the change + * (expected v2.02.000, DEV-1011). Because nothing parses on it, a wrong + * value here can only mis-word a warning. + */ + public static final ShimmerVerObject CCF_GEN2_LIGHT_RATE = new ShimmerVerObject(FW_ID.UNKNOWN, 2, 2, 0); } public static class FW_SPECIAL_VERSIONS { @@ -723,6 +742,14 @@ public boolean isPayloadDesignV13orAbove() { return PayloadContentsDetails.isPayloadDesignV13orAbove(getShimmerVerObject()); } + /** + * See {@link FW_CHANGES#CCF_GEN2_LIGHT_RATE} - diagnostic only, no parsing + * decision keys on this. + */ + public boolean isPayloadDesignV14orAbove() { + return PayloadContentsDetails.isPayloadDesignV14orAbove(getShimmerVerObject()); + } + /** * Whether a given Verisense hardware revision is second-generation * (SR68-9/10, SR61-5/6). Mirrors the TypeScript SDK @@ -1062,21 +1089,33 @@ && isAnyLsm6dsvChannelEnabled()) { sb.append("}"); } else if(sensorClassKey==AbstractSensor.SENSORS.VD6283 && isSensorEnabled(Configuration.Verisense.SENSOR_ID.VD6283)) { - // Second-generation ambient light. The configured sample rate isn't stored - // in the payload header, so only the calculated (achieved) rate is reported. + // Second-generation ambient light. From FW v2.02.000 the configured rate + // is in the payload header (byte 30 bits 6:3) and is reported like every + // other sensor; for earlier recordings it was stored nowhere, so only the + // calculated (achieved) rate can be given. SensorVD6283 sensorVd6283 = getSensorVD6283(); - sb.append(sensorClassKey.toString()); - sb.append(" {Sampling Rate ["); - sb.append(SENSOR_CONFIG_STRINGS.SAMPLING_RATE_CALCULATED); - if(!Double.isNaN(calculatedSamplingRate)) { - sb.append(UtilVerisenseDriver.formatDoubleToNdecimalPlaces(calculatedSamplingRate, 3)); - sb.append(" "); - sb.append(CHANNEL_UNITS.FREQUENCY); + if(sensorVd6283.isConfiguredRateKnown()) { + // The CONFIGURED rate, not getRateFreq()'s exposure-clamped one. This is + // the field a user reads to check what they asked the device to do, so a + // 20 Hz configuration must not print as 10 Hz merely because the exposure + // cannot sustain it. The clamp still governs block timing and the gap + // window; Exposure is on this same line for anyone deriving the bound. + sb.append(generateCalcSamplingRateConfigStr(sensorClassKey, sensorVd6283.getRate().freqHz, calculatedSamplingRate)); } else { - sb.append(UtilVerisenseDriver.UNAVAILABLE); + sb.append(sensorClassKey.toString()); + sb.append(" {Sampling Rate ["); + sb.append(SENSOR_CONFIG_STRINGS.SAMPLING_RATE_CALCULATED); + if(!Double.isNaN(calculatedSamplingRate)) { + sb.append(UtilVerisenseDriver.formatDoubleToNdecimalPlaces(calculatedSamplingRate, 3)); + sb.append(" "); + sb.append(CHANNEL_UNITS.FREQUENCY); + } else { + sb.append(UtilVerisenseDriver.UNAVAILABLE); + } + sb.append("]; "); } - sb.append("]; Gain = "); + sb.append("Gain = "); sb.append(UtilVerisenseDriver.formatDoubleToNdecimalPlaces(sensorVd6283.getGain(), 2)); sb.append("x; Exposure = "); sb.append(UtilVerisenseDriver.formatDoubleToNdecimalPlaces(sensorVd6283.getExposureUs()/1000.0, 1)); diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/AsmBinaryFileConstants.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/AsmBinaryFileConstants.java index 778cc7e24..3407f490f 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/AsmBinaryFileConstants.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/AsmBinaryFileConstants.java @@ -42,8 +42,10 @@ public class PAYLOAD_CONFIG_BYTE_INDEX { // Second-generation (payload design v13) only: GEN_CFG_3 at abs byte 29 (rel 25) // carries the mag/light/skin-temp/algo-hub enables + LED mode. public static final int GEN_CFG_3 = 25; - /** Second-generation only: ambient-light gain index in bits 2:0 and the - * dark-channel enable in bit 7 (abs byte 30). */ + /** Second-generation only (abs byte 30): ambient-light gain index in bits + * 2:0, the effective sample-rate index in bits 6:3 from FW v2.02.000 where + * 0 means not recorded, and the dark-channel enable in bit 7. See + * {@code SensorVD6283.VD6283_RATE}. */ public static final int LIGHT_GAIN_AND_DARK = 26; /** Second-generation only: ambient-light exposure index (abs byte 31). */ public static final int LIGHT_EXPOSURE = 27; diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetails.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetails.java index 8495799a2..2e869149b 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetails.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetails.java @@ -277,6 +277,17 @@ public static boolean isPayloadDesignV13orAbove(ShimmerVerObject svo) { return VerisenseDevice.compareFwVersions(svo, VerisenseDevice.FW_CHANGES.CCF_GEN2); } + /** + * VD6283 effective sample-rate index present in payload header byte 30 bits + * 6:3. See {@link VerisenseDevice.FW_CHANGES#CCF_GEN2_LIGHT_RATE}: this is a + * DIAGNOSTIC marker only. The header length is unchanged at this version, so + * nothing about where fields are read from depends on it, and the rate field + * itself says whether it is populated. + */ + public static boolean isPayloadDesignV14orAbove(ShimmerVerObject svo) { + return VerisenseDevice.compareFwVersions(svo, VerisenseDevice.FW_CHANGES.CCF_GEN2_LIGHT_RATE); + } + /** * This is a similar method to "!isPayloadDesignGen7OrAbove" but has been * created to make it easy to distinguish in the code sections that control the diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java index 9943a8481..254b95ae6 100644 --- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java +++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/payloaddesign/PayloadContentsDetailsV8orAbove.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.ListIterator; import java.util.Map.Entry; @@ -93,13 +92,30 @@ public void parsePayloadContentsMetaData(int binFileByteIndex) throws IOExceptio // --------- End of parsing ------------------ - // The slow sensors' achieved sample rates differ from what the payload - // header can tell us (the light rate isn't stored at all and the chip adds - // per-measurement dead time; the skin-temp output cadence is refresh-code - // derived but similarly approximate), so refine them from the data itself - // before the block timings are back-filled below. - refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID.LIGHT); - refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID.SKIN_TEMP); + // The slow sensors carry their configured sample rate in the payload header + // (MLX90632 refresh code in byte 32; VD6283 rate index in byte 30 bits 6:3 + // from FW v2.02.000), so where the rate is known the blocks are already + // correctly timed and only the CSV gap window has to be derived from it - + // see the method javadoc for why this is no longer measured from the data. + // + // Where it is NOT known - an ambient light recording from firmware earlier + // than v2.02.000 - the blocks are timed from the exposure bound, which only + // bounds the rate from above and is ten times too fast at the 1 Hz default. + // That is a real loss against measuring the spacing, and it is deliberate: + // measuring cost the ability to report data loss at all. + // + // Be precise about what it costs, because it is more than the CSV header. + // The light and skin-temp CSVs carry no timestamp column, so the FILE PARSER + // output moves only by its header start time, up to 8.1 s at 1 Hz. But the + // per-sample ObjectCluster stream is stamped from each block's start time + // stepping by its timestampDiffInS, so for those recordings every consumer + // of that stream - the Android API, live streaming, algorithm modules - now + // sees a 10-sample block laid over 0.9 s of the 10 s it really spans, and a + // 9.1 s jump to the next block. Second-generation firmware never shipped to + // a customer, so this reaches internal recordings only, and a recording from + // v2.02.000 onwards is unaffected either way. + seedSlowSensorGapWindow(DATABLOCK_SENSOR_ID.LIGHT); + seedSlowSensorGapWindow(DATABLOCK_SENSOR_ID.SKIN_TEMP); // Up to, and including, payload design v10, the real-world clock time that was // stored in the payload footer was the real-world time at the end of the @@ -316,95 +332,55 @@ private boolean isParserAtEndOfBuffer(int bufferLength, int currentByteIndex) { } /** - * Derive a slow sensor's (ambient light / skin temp) achieved sample period - * from the spacing of consecutive same-sensor block end ticks and apply it to - * those blocks' sampling rate before their timings are back-filled. The - * header-derived rates are only estimates (the light rate isn't stored at all - * - see SensorVD6283.getRateFreq - and the skin-temp cadence is refresh-code - * derived), and each block holds a fixed number of samples, so - * {@code inter-block ticks / samples-per-block} is the exact per-sample period. - * With fewer than two blocks in the payload the header-derived estimate the - * blocks were created with is left in place. + * Set a slow sensor CSV gap-splitting window from the rate the payload header + * states, before the block timings are back-filled below. + *
+ * Both slow sensors now carry their configured rate in the header - the + * MLX90632 refresh code always did (byte 32), and the VD6283 rate index does + * from FW v2.02.000 (byte 30 bits 6:3) - so the blocks keep the rate + * {@code parseDataBlockMetaData} already seeded them with from + * {@code getSamplingRateForSensor}, and nothing here re-times them. That is + * what makes the FIRST block of a CSV set correctly timed rather than laid + * out over a guessed span. + *
+ * This deliberately replaces the older approach of measuring the period from
+ * the spacing of consecutive same-sensor block end ticks. Measuring works
+ * only on healthy data: a payload holds two or three slow-sensor blocks, so
+ * the window was built from one or two gaps, and if one of those was itself a
+ * dropped block it became the median and the gap was judged continuous while
+ * the healthy boundary beside it was flagged instead (DEV-974). A window
+ * derived from the configured rate cannot be pulled onto a wrong cadence by
+ * the very loss it exists to detect.
+ *
+ * @param slowSensorId the slow sensor data block id
*/
- private void refineSlowSensorSamplingRateFromBlockTicks(DATABLOCK_SENSOR_ID slowSensorId) {
- List
+ * This also sets the MLX90632 gap edge, and that edge is pinned against
+ * real data. For skin temp the window is widened again by
+ * {@link #SLOW_SENSOR_CONVERSION_SLIP_TOLERANCE}, so the edge lands at
+ * {@code cfg/1.725} - and Test_065 contains a HEALTHY skin-temp boundary at
+ * 1.63x nominal spacing during start-up settling. Retuning this constant for
+ * the light sensor alone moves the skin-temp edge with it: 1.4 would put it
+ * at 1.61x and re-split that recording. See
+ * {@link UtilCsvSplitting#getSlowSensorPlausibleRateRangeHz} and change it
+ * only against a measurement on both sensors.
*/
public static final double SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO = 1.5;
+ /**
+ * MLX90632 only: how far a SINGLE block boundary apparent rate may sit
+ * either side of the configured output rate and still be plausible.
+ *
+ * The chip conversions slip by several refresh periods and then catch up
+ * (+12.5% block spacing observed on the DEV-927 validation recording, with
+ * no samples lost), so one boundary reads ~12.5% slow and the next
+ * correspondingly fast. Only the AVERAGE rate is bounded by the configured
+ * one, which is why the configured rate alone is too tight a bound. 1.15
+ * covers the observed slip with a little margin.
+ *
+ * Not used for the VD6283, whose sampling is a plain periodic timer with no
+ * slip-and-catch-up behaviour: a failed read there costs a whole period, it
+ * never shortens one.
+ */
+ public static final double SLOW_SENSOR_CONVERSION_SLIP_TOLERANCE = 1.15;
}
protected static HashMap
+ * VD6283: true once the firmware stores the rate index in header byte 30
+ * bits 6:3 (FW v2.02.000+). MLX90632: always true for an enabled gen-2
+ * device, because the refresh code has always been in header byte 32.
+ *
+ * @param verisenseDevice the device being parsed
+ * @param slowSensorId the slow sensor data block id
+ * @return true when the configured rate is known from the header alone
+ */
+ public static boolean isSlowSensorConfiguredRateKnown(VerisenseDevice verisenseDevice, DATABLOCK_SENSOR_ID slowSensorId) {
+ if(slowSensorId==DATABLOCK_SENSOR_ID.LIGHT) {
+ SensorVD6283 sensorVd6283 = verisenseDevice.getSensorVD6283();
+ return sensorVd6283!=null && sensorVd6283.isConfiguredRateKnown();
+ }
+ if(slowSensorId==DATABLOCK_SENSOR_ID.SKIN_TEMP) {
+ return isRateUsable(verisenseDevice.getSamplingRateForSensor(SENSORS.MLX90632));
+ }
+ return false;
+ }
+
+ private static boolean isRateUsable(double rateHz) {
+ return rateHz>0 && !Double.isNaN(rateHz) && !Double.isInfinite(rateHz);
+ }
+
+ /**
+ * The {min, max} per-sample rate a slow sensor could legitimately be running
+ * at, which the CSV gap window is then built from.
+ *
+ * When the configured rate is known from the header the range collapses onto
+ * it - exactly for the VD6283, whose timer has no slip behaviour, and widened
+ * by {@link FILE_GAP_TOLERANCE_MULTIPLIER#SLOW_SENSOR_CONVERSION_SLIP_TOLERANCE}
+ * for the MLX90632, whose conversions slip and catch up.
+ *
+ * When it is not known - a light recording from FW earlier than v2.02.000 -
+ * the range is the whole firmware rate table. That is deliberately wide:
+ * [0.5, 20] Hz becomes a [0.33, 22] Hz window, so a 10-sample light block
+ * boundary is accepted anywhere up to 30 s and genuine loss goes unreported.
+ * It is the best that can be done without the rate, it keeps such recordings
+ * in ONE CSV rather than one per block, and it is stateless: nothing is
+ * learned from the data, so no amount of unhealthy data can move it.
+ *
+ * What comes back is a plausible RATE range. The gap window built from it is
+ * wider again on both sides - see {@link #seedSlowSensorGapWindow}, which also
+ * explains why the MLX90632 edge emerging from
+ * {@link FILE_GAP_TOLERANCE_MULTIPLIER#SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO}
+ * cannot be retuned for the light sensor alone.
+ *
+ * @param verisenseDevice the device being parsed
+ * @param slowSensorId the slow sensor data block id
+ * @return the {min, max} plausible rate in Hz, or null if it cannot be bounded
+ */
+ public static double[] getSlowSensorPlausibleRateRangeHz(VerisenseDevice verisenseDevice, DATABLOCK_SENSOR_ID slowSensorId) {
+ if(slowSensorId==DATABLOCK_SENSOR_ID.LIGHT) {
+ if(isSlowSensorConfiguredRateKnown(verisenseDevice, slowSensorId)) {
+ double configuredRateHz = verisenseDevice.getSensorVD6283().getRateFreq();
+ if(isRateUsable(configuredRateHz)) {
+ return new double[] {configuredRateHz, configuredRateHz};
+ }
+ }
+ return new double[] {SensorVD6283.MIN_SAMPLE_RATE_HZ, SensorVD6283.MAX_SAMPLE_RATE_HZ};
+ }
+ if(slowSensorId==DATABLOCK_SENSOR_ID.SKIN_TEMP) {
+ double configuredRateHz = verisenseDevice.getSamplingRateForSensor(SENSORS.MLX90632);
+ if(isSlowSensorConfiguredRateKnown(verisenseDevice, slowSensorId)) {
+ // Both sides are widened, and the slow side is the uncomfortable one.
+ // It puts the gap edge at cfg/1.725, while a dropped block landing on a
+ // 12.5% catch-up presents cfg/1.75 - so that case is reported, but by
+ // 1.4%, and it stops being reported at all once a catch-up reaches
+ // 13.75%.
+ //
+ // Narrowing the slow side to cfg/1.5, as the VD6283 uses, was tried and
+ // reverted: the Test_065 recording contains a healthy skin-temp boundary
+ // at 1.63x nominal spacing during start-up settling, which then split. So
+ // this sensor's healthy behaviour genuinely overlaps the region a dropped
+ // block would land in, and no single threshold separates them cleanly.
+ // 1.725 is the midpoint that was chosen with that data in hand: 6% above
+ // the worst healthy boundary observed, 14% below a clean dropped block.
+ // Moving it in either direction trades false splits against missed loss,
+ // so change it only against a measurement, not an intuition.
+ return new double[] {
+ configuredRateHz/FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_CONVERSION_SLIP_TOLERANCE,
+ configuredRateHz*FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_CONVERSION_SLIP_TOLERANCE};
+ }
+ // Unreachable today: getRateFreq() is refresh/sub-measurements, always
+ // within [0.167, 32], and the sensor class exists whenever skin-temp
+ // blocks do. Kept because the caller no longer guarantees either.
+ return new double[] {SensorMLX90632.MIN_OUTPUT_RATE_HZ, SensorMLX90632.MAX_OUTPUT_RATE_HZ};
+ }
+ return null;
+ }
+
+ /**
+ * Sets a slow sensor CSV gap-splitting window from the rate the payload
+ * header states, replacing the configured-rate +/-10% band that
+ * {@link #populateExpectedPayloadTsDiffLimitMapIfNeeded} would otherwise
+ * install.
+ *
+ * The gap side is {@code min / SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO} so
+ * healthy jitter stays continuous while a dropped block, which doubles the
+ * spacing and so halves the apparent rate, is reported. The fast side is
+ * {@code max * UPPER}, which catches an overlapping block or a backwards
+ * clock step. Worked through at 1 Hz the window is [0.667, 1.1] Hz: a dropped
+ * block presents 0.5 Hz and splits, a block that took one extra period after
+ * a failed I2C read presents 0.909 Hz and does not.
+ *
+ * The VD6283 gets a clean 25% margin on a dropped block. The MLX90632 does
+ * not, and cannot: its window is {@code [f/1.725, f*1.265]}, so a dropped
+ * block presents {@code 0.5f} and splits comfortably, but a dropped block
+ * landing on the chip's documented 12.5% catch-up presents {@code 0.571f}
+ * against an edge of {@code 0.580f} and splits by 1.4%. That is not slack
+ * left lying around - see {@link #getSlowSensorPlausibleRateRangeHz} for the
+ * measurement that pins the edge where it is.
+ *
+ * Nothing here depends on previously seen data, so unlike a measured window
+ * this cannot be pulled onto a wrong cadence by the very loss it is meant to
+ * detect, and it is correct from the FIRST boundary of a CSV set rather than
+ * after several. The put is unconditional because
+ * populateExpectedPayloadTsDiffLimitMapIfNeeded is containsKey-guarded and
+ * runs later in the parse: this window must win.
+ *
+ * The MLX90632 gap edge is coupled to
+ * {@link FILE_GAP_TOLERANCE_MULTIPLIER#SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO}
+ * and it is pinned against real data. That edge is
+ * {@code cfg / (1.15 x 1.5) = cfg/1.725}, an emergent product of the slip
+ * tolerance and that ratio, and Test_065 contains a healthy skin-temp boundary
+ * at 1.63x nominal spacing. Retuning the ratio for the light sensor alone
+ * moves the skin-temp edge with it: 1.4 would put it at 1.61x and re-split
+ * that recording. Check both sensors, against a measurement.
+ *
+ * @param verisenseDevice the device being parsed
+ * @param slowSensorId the slow sensor data block id
+ */
+ public static void seedSlowSensorGapWindow(VerisenseDevice verisenseDevice, DATABLOCK_SENSOR_ID slowSensorId) {
+ double[] plausibleRateRangeHz = getSlowSensorPlausibleRateRangeHz(verisenseDevice, slowSensorId);
+ if(plausibleRateRangeHz==null) {
+ return;
+ }
+ double[] samplingRateLimits = new double[] {
+ plausibleRateRangeHz[0]/FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO,
+ plausibleRateRangeHz[1]*FILE_GAP_TOLERANCE_MULTIPLIER.UPPER};
+ for(SENSORS sensorClassKey:verisenseDevice.getOrCreateListOfSensorClassKeysForDataBlockId(slowSensorId)) {
+ if(sensorClassKey!=SENSORS.CLOCK) {
+ SAMPLING_RATE_LIMITS_PER_SENSOR.put(sensorClassKey, samplingRateLimits);
+ }
+ }
+ }
+
+ /**
+ * Set once an unusable ambient-light rate field has been reported, so it is
+ * one line rather than one per payload.
+ *
+ * Once per CSV SET, not once per recording: it is cleared with the limits map,
+ * and {@code AsmBinaryFileParse.writeAllStringBuildersToFiles()} clears that at
+ * every CSV split - config change and time-based alike - as well as at the
+ * start and end of a file. A recording split into four CSV sets can therefore
+ * print this four times, which is the right granularity anyway: each set is a
+ * separate CSV that somebody may open on its own.
+ */
+ private static boolean hasWarnedLightRateFieldUnusable = false;
+
public static void clearMapOfSamplingRateLimitsPerSensor() {
SAMPLING_RATE_LIMITS_PER_SENSOR.clear();
+ hasWarnedLightRateFieldUnusable = false;
+ }
+
+ /**
+ * Reports, once per CSV set, an ambient-light rate field this parser cannot
+ * use: either a payload claiming firmware new enough to store the index yet
+ * leaving it clear, or any payload holding a reserved index 7..15.
+ *
+ * This is the one failure the header-driven design cannot otherwise see. A
+ * zero field is indistinguishable from an old recording, so the parser falls
+ * back to the wide rate-table window, the light data still comes out, and
+ * nothing anywhere says that the gap detection for this file is nearly blind
+ * - see {@link #getSlowSensorPlausibleRateRangeHz} for how wide that fallback
+ * is. If the firmware ever ships with the field broken, this line is the only
+ * thing that will say so.
+ *
+ * Nothing about parsing keys on the firmware version: the field is
+ * self-describing and {@link SensorVD6283#isConfiguredRateKnown()} alone
+ * decides behaviour. The version is read HERE and nowhere else, purely to tell
+ * "old recording, as expected" apart from "new recording, firmware bug". A
+ * wrong value in {@link VerisenseDevice.FW_CHANGES#CCF_GEN2_LIGHT_RATE} can
+ * therefore only make this warning fire at the wrong boundary; it cannot
+ * change a parse.
+ *
+ * @param verisenseDevice the device being parsed
+ */
+ public static void warnIfLightRateFieldUnusable(VerisenseDevice verisenseDevice) {
+ if(hasWarnedLightRateFieldUnusable) {
+ return;
+ }
+ SensorVD6283 sensorVd6283 = verisenseDevice.getSensorVD6283();
+ if(sensorVd6283==null || sensorVd6283.isConfiguredRateKnown()) {
+ return;
+ }
+
+ // Two different faults, and they must not be reported as one.
+ String fault;
+ if(sensorVd6283.isRateIndexReserved()) {
+ // A value no firmware rate table defines. Report this whatever the
+ // payload design claims: on an old recording those bits should be clear,
+ // so a non-zero one is an anomaly in its own right and the version tells
+ // us nothing useful about it.
+ //
+ // Deliberately says nothing about the recording carrying light blocks. The
+ // caller in PayloadContentsDetailsV8orAbove only reaches here for a payload
+ // that does, but this method is public and does not check, so stating it
+ // would be a claim made on somebody else's behalf.
+ fault = "wrote the reserved index " + sensorVd6283.getRateIndexRaw()
+ + " into the ambient-light rate field (payload header byte 30 bits 6:3),"
+ + " which no firmware rate table defines";
+ } else if(verisenseDevice.isPayloadDesignV14orAbove()) {
+ fault = "stores the VD6283 sample rate in payload header byte 30 bits 6:3, but this"
+ + " recording carries ambient light blocks with that field clear. This is a"
+ + " firmware fault, not an old recording";
+ } else {
+ // Old recording, field clear: exactly as expected, and not worth a line.
+ return;
+ }
+
+ hasWarnedLightRateFieldUnusable = true;
+ double[] fallbackRangeHz = getSlowSensorPlausibleRateRangeHz(verisenseDevice, DATABLOCK_SENSOR_ID.LIGHT);
+ System.out.println("WARNING!!! Firmware " + verisenseDevice.getFirmwareVersionParsed() + " "
+ + fault + ". Falling back to the whole rate table, " + fallbackRangeHz[0] + " to "
+ + fallbackRangeHz[1] + " Hz, so light blocks are timed from the exposure bound"
+ + " and gap detection for this file is close to blind.");
}
public static String isDataBlockContinuous(SENSORS sensorClassKey, DataSegmentDetails dataSegmentDetailsPrevious, DataBlockDetails nextDataBlockDetails) {
diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorMLX90632.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorMLX90632.java
index edcf0cd56..bed0d4ee2 100644
--- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorMLX90632.java
+++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorMLX90632.java
@@ -36,8 +36,10 @@
* refresh-rate code (0..7 = 0.5/1/2/4/8/16/32/64 Hz). The OUTPUT sample rate
* the firmware delivers is refresh / sub-measurements (medical = 2,
* extended = 3) - mirrors the web SDK's SensorMLX90632.ts. As with the
- * ambient light, the parser refines the achieved rate per payload from
- * temp-block tick spacing; the header-derived value seeds the timing.
+ * ambient light, the header-derived value IS the timing: nothing is measured
+ * back out of the data. Refining the achieved rate from temp-block tick spacing
+ * was removed with DEV-974, because it could only work on healthy data and so
+ * went blind exactly when a recording had lost samples.
*/
public class SensorMLX90632 extends AbstractSensor {
@@ -55,6 +57,19 @@ public class SensorMLX90632 extends AbstractSensor {
public static final int SUB_MEASUREMENTS_MEDICAL = 2;
public static final int SUB_MEASUREMENTS_EXTENDED = 3;
+ /**
+ * Output-rate bounds implied by the refresh table across BOTH modes: the
+ * slowest configuration is REFRESH_HZ_TABLE[0] (0.5 Hz) divided by
+ * {@link #SUB_MEASUREMENTS_EXTENDED} (3) = 0.167 Hz, and the fastest is
+ * REFRESH_HZ_TABLE[7] (64 Hz) divided by {@link #SUB_MEASUREMENTS_MEDICAL}
+ * (2) = 32 Hz. Unlike the VD6283 before FW v2.02.000, this sensor refresh
+ * code has always been stored in the payload header, so {@link
+ * #getRateFreq()} is the real configured output rate and these bounds are
+ * only needed as a fallback when that rate cannot be read.
+ */
+ public static final double MIN_OUTPUT_RATE_HZ = 0.5/3;
+ public static final double MAX_OUTPUT_RATE_HZ = 32.0;
+
private int refreshRateCode = 5; // 16 Hz chip refresh (firmware default)
private boolean extendedMode = false;
@@ -174,9 +189,14 @@ public String getMeasTypeString() {
/**
* Header-derived output sample rate (Hz): the chip refresh rate divided by
- * the sub-measurements per output (medical = 2, extended = 3). Seeds
- * data-block timing; the parser refines the achieved rate per payload from
- * temp-block tick spacing.
+ * the sub-measurements per output (medical = 2, extended = 3). This times the
+ * data blocks directly; nothing refines it from the data afterwards.
+ *
+ * It is the CONFIGURED rate. Conversions slip by several refresh periods and
+ * then catch up, so an individual boundary can read up to about 12.5% either
+ * side of it while losing no samples at all - which is why the CSV gap window
+ * built from this widens BOTH sides by
+ * {@code SLOW_SENSOR_CONVERSION_SLIP_TOLERANCE}.
*/
public double getRateFreq() {
return getRefreshHz() / (extendedMode ? SUB_MEASUREMENTS_EXTENDED : SUB_MEASUREMENTS_MEDICAL);
diff --git a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorVD6283.java b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorVD6283.java
index 37ec8437b..4bf431bd0 100644
--- a/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorVD6283.java
+++ b/ShimmerDriver/src/main/java/com/shimmerresearch/verisense/sensors/SensorVD6283.java
@@ -43,14 +43,21 @@
* and BLUE are unaffected, so lux/CCT remain valid in either mode. No
* dark-enabled recording exists yet - revisit the naming when one does.
*
- * The sample rate is configured in the op config (LIGHT_SAMPLE_RATE_INDEX) but
- * is NOT mirrored into the stored payload header, and the achieved rate also
- * differs from both the configured rate and 1/exposure (the chip adds dead
- * time per measurement, e.g. ~110 ms/sample for the 100 ms exposure). The
- * getRateFreq() value here is therefore only the exposure-limited ESTIMATE
- * used to seed data-block timing; the parser refines it per payload from the
- * spacing of consecutive light-block timestamps
- * (PayloadContentsDetailsV8orAbove.refineLightSamplingRateFromBlockTicks).
+ * The sample rate is configured in the op config (LIGHT_SAMPLE_RATE_INDEX).
+ * From FW v2.02.000 the firmware mirrors the EFFECTIVE index (post-default)
+ * into payload header byte 30 bits 6:3, so {@link #getRateFreq()} returns the
+ * real configured rate - see {@link VD6283_RATE}. Earlier firmware stored it
+ * nowhere and left 0 in those bits, in which case all that can be said is that
+ * 1/exposure bounds the rate from ABOVE; at the firmware default of 1 Hz with
+ * the default 100 ms exposure that bound is ten times the truth, which is what
+ * fragmented light data into one CSV per block (DEV-979).
+ *
+ * Even the configured rate is not always the ACHIEVED rate: the chip measures
+ * every max(inter-measurement, exposure) plus dead time, so 10 Hz configured
+ * with a 100 ms exposure achieves ~9.09 Hz. getRateFreq() therefore clamps the
+ * configured rate with the exposure bound, and the CSV gap window built from it
+ * (UtilCsvSplitting.seedSlowSensorGapWindow) is tolerant enough to absorb the
+ * remainder.
*/
public class SensorVD6283 extends AbstractSensor {
@@ -81,7 +88,113 @@ public class SensorVD6283 extends AbstractSensor {
{-0.028752, 0.506372, -0.120614},
{-0.552625, 0.335866, 0.494781}};
- /** Poll ceiling in continuous mode (firmware slow-sensor sampler). */
+ /**
+ * The rates the firmware can be configured to, from the slow-sensor sampler
+ * index table (hal_slowSensorSampler.c {@code slowSensorRateMs[] = {0, 2000,
+ * 1000, 500, 200, 100, 50}} ms). The index lives in operational-config byte
+ * 75 (LIGHT_SAMPLE_RATE_INDEX) and, from FW v2.02.000, is mirrored into
+ * payload header byte 30 bits 6:3 as the EFFECTIVE index - the value the
+ * sampler was actually started with, after the firmware fallback from 0
+ * (Off) to 1 Hz. It is therefore never {@code NOT_STORED} while light data
+ * blocks exist, which is what lets a parser read 0 as "recorded by earlier
+ * firmware" without consulting the firmware version at all.
+ *
+ * The indices are a stored-data contract shared with the firmware: they may
+ * be appended to but never renumbered, and cannot exceed the 4-bit header
+ * field (the firmware carries a compile-time guard for that).
+ */
+ public static enum VD6283_RATE {
+ NOT_STORED("Not stored", 0, 0.0),
+ /**
+ * Bits 6:3 held 7..15, which the firmware rate table does not define.
+ *
+ * Distinct from {@link #NOT_STORED} on purpose. Both end up with the wide
+ * fallback window, but they mean opposite things: NOT_STORED is an old
+ * recording behaving exactly as expected, while this is a payload that
+ * wrote a rate nobody can decode. Collapsing the two made the firmware-fault
+ * warning report a field that was "clear" when it was not, which sends a
+ * reader looking for the wrong bug.
+ *
+ * Not in {@code BY_CONFIG_VALUE}: -1 is not a storable index.
+ */
+ RESERVED_INDEX("Reserved", -1, 0.0),
+ RATE_0_5_HZ("0.5Hz", 1, 0.5),
+ RATE_1_HZ("1.0Hz", 2, 1.0),
+ RATE_2_HZ("2.0Hz", 3, 2.0),
+ RATE_5_HZ("5.0Hz", 4, 5.0),
+ RATE_10_HZ("10.0Hz", 5, 10.0),
+ RATE_20_HZ("20.0Hz", 6, 20.0);
+
+ public String label;
+ public Integer configValue;
+ public double freqHz;
+
+ static Map
+ * The question is answered here rather than at each call site because it was
+ * being asked in two places with two different answers: the driver tested
+ * both sentinels, while the ASM_PC header patcher tested only NOT_STORED, so
+ * adding RESERVED_INDEX let an undecodable index through the patcher and into
+ * a regression dataset. One predicate, one place to update when a member is
+ * added.
+ *
+ * Both properties are tested because a real rate has both - a storable index
+ * 1..6 and a non-zero frequency - and each verdict has neither. A member
+ * carrying one without the other would be a mistake in this enum, and
+ * unusable is the safe way to read a mistake.
+ */
+ public boolean isStorable() {
+ return configValue>0 && freqHz>0;
+ }
+ }
+
+ /** Raw bits 6:3 of header byte 30, kept so diagnostics can name what was there. */
+ private int rateIndexRaw = 0;
+
+ /** Payload header byte 30 field layout (firmware PAYLOAD_HDR_LIGHT_*). */
+ public static final int LIGHT_GAIN_MASK = 0x07;
+ public static final int LIGHT_RATE_INDEX_BIT_SHIFT = 3;
+ public static final int LIGHT_RATE_INDEX_MASK = 0x0F;
+ public static final int LIGHT_DARK_ENABLE_MASK = 0x80;
+
+ /** Slowest rate the firmware can be configured to (see VD6283_RATE). */
+ public static final double MIN_SAMPLE_RATE_HZ = 0.5;
+ /** Fastest, and also the poll ceiling in continuous mode. */
public static final double MAX_SAMPLE_RATE_HZ = 20.0;
public static final String UNITS_LUX = "lux";
@@ -90,6 +203,7 @@ public class SensorVD6283 extends AbstractSensor {
private int gainIndex = 0;
private int exposureIndex = 0;
private boolean darkChannelEnabled = false;
+ private VD6283_RATE rate = VD6283_RATE.NOT_STORED;
public class GuiLabelSensors {
public static final String LIGHT = "Light";
@@ -283,9 +397,43 @@ public double[] computeLuxCct(double red, double green, double blue) {
public void configBytesParse(ShimmerDevice shimmerDevice, byte[] configBytes, COMMUNICATION_TYPE commType) {
if(commType == COMMUNICATION_TYPE.SD && isSensorEnabled(Configuration.Verisense.SENSOR_ID.VD6283)) {
int gainAndDark = configBytes[PAYLOAD_CONFIG_BYTE_INDEX.LIGHT_GAIN_AND_DARK] & 0xFF;
- gainIndex = gainAndDark & 0x07;
- darkChannelEnabled = (gainAndDark & 0x80) != 0;
+ gainIndex = gainAndDark & LIGHT_GAIN_MASK;
+ darkChannelEnabled = (gainAndDark & LIGHT_DARK_ENABLE_MASK) != 0;
+ // Bits 6:3 were spare before FW v2.02.000 and were written as zero, so a
+ // zero here means the rate was not recorded rather than index 0. No
+ // firmware-version check is needed: the firmware stores the EFFECTIVE
+ // index, which is never zero while light blocks exist.
+ //
+ // "Earlier firmware always left these clear" is load-bearing, so here is
+ // the proof rather than the assertion. On firmware before 2.02.000,
+ // backupConfigSettings() built this byte as
+ // cfg15Bkup = (lightGainIndex & 0x07) | (lightDarkEnable ? 0x80 : 0)
+ // which cannot set bits 6:3, and resetPayloadBackupConfig() zeroed the
+ // whole byte. There is no third writer.
+ //
+ // Gating this on the firmware version as well would look safer and is
+ // not: the header-patched regression datasets deliberately keep their
+ // true firmware version, so a version gate would stop the parser reading
+ // the very field those datasets exist to exercise. A reserved index is
+ // the residual risk and it is reported rather than silently accepted -
+ // see VD6283_RATE.RESERVED_INDEX.
+ rateIndexRaw = (gainAndDark >> LIGHT_RATE_INDEX_BIT_SHIFT) & LIGHT_RATE_INDEX_MASK;
+ rate = VD6283_RATE.getForConfigValue(rateIndexRaw);
exposureIndex = configBytes[PAYLOAD_CONFIG_BYTE_INDEX.LIGHT_EXPOSURE] & 0xFF;
+ } else if(commType == COMMUNICATION_TYPE.SD) {
+ // The light sensor is not enabled in THIS payload, so there is no rate to
+ // read. Defence in depth only: today this cannot find a stale value to
+ // clear, because VerisenseDevice.configBytesParse calls
+ // sensorAndConfigMapsCreate() before any sensor config is parsed, and that
+ // replaces this object with a fresh one. Sensor state does NOT carry from
+ // payload to payload - do not reason as though it does.
+ //
+ // Kept rather than deleted because the invariant it leans on belongs to
+ // another class. Were instances ever reused across payloads, a stale rate
+ // would be worse than a stale gain or exposure: those only affect
+ // calibration, while the rate drives block timing and CSV splitting.
+ rateIndexRaw = 0;
+ rate = VD6283_RATE.NOT_STORED;
}
}
@@ -307,13 +455,50 @@ public boolean isDarkChannelEnabled() {
}
/**
- * Exposure-limited sample-rate ESTIMATE (Hz). The configured rate is not in
- * the stored payload header and the chip adds per-measurement dead time, so
- * this only seeds data-block timing - the parser refines the rate per payload
- * from consecutive light-block timestamps.
+ * The sample rate to time this sensor data blocks with (Hz).
+ *
+ * When the payload header carries the configured rate (FW v2.02.000+, see
+ * {@link VD6283_RATE}) that rate is returned, clamped by the exposure bound
+ * because the chip cannot measure faster than it integrates. Otherwise only
+ * the exposure bound itself is available, and that is an UPPER BOUND rather
+ * than an estimate of the rate: it is ten times too fast at the firmware
+ * default of 1 Hz. Callers that need to know which of the two they got
+ * should ask {@link #isConfiguredRateKnown()}.
*/
public double getRateFreq() {
- return Math.min(MAX_SAMPLE_RATE_HZ, 1e6 / getExposureUs());
+ double exposureLimitHz = Math.min(MAX_SAMPLE_RATE_HZ, 1e6 / getExposureUs());
+ if(isConfiguredRateKnown()) {
+ return Math.min(rate.freqHz, exposureLimitHz);
+ }
+ return exposureLimitHz;
+ }
+
+ /**
+ * Whether the payload header told us the configured sample rate. False for
+ * recordings from FW earlier than v2.02.000, where {@link #getRateFreq()}
+ * can only return the exposure bound.
+ */
+ public boolean isConfiguredRateKnown() {
+ return rate.isStorable();
+ }
+
+ /**
+ * True when bits 6:3 held a value the firmware rate table does not define.
+ *
+ * Worth separating from "not stored" because it should never happen and says
+ * something different when it does - see {@link VD6283_RATE#RESERVED_INDEX}.
+ */
+ public boolean isRateIndexReserved() {
+ return rate==VD6283_RATE.RESERVED_INDEX;
+ }
+
+ /** The raw 4-bit value from header byte 30 bits 6:3, for diagnostics. */
+ public int getRateIndexRaw() {
+ return rateIndexRaw;
+ }
+
+ public VD6283_RATE getRate() {
+ return rate;
}
@Override
diff --git a/ShimmerDriver/src/test/java/com/shimmerresearch/verisense/payloaddesign/API_00011_VerisenseSlowSensorGapWindow.java b/ShimmerDriver/src/test/java/com/shimmerresearch/verisense/payloaddesign/API_00011_VerisenseSlowSensorGapWindow.java
new file mode 100644
index 000000000..3af103536
--- /dev/null
+++ b/ShimmerDriver/src/test/java/com/shimmerresearch/verisense/payloaddesign/API_00011_VerisenseSlowSensorGapWindow.java
@@ -0,0 +1,811 @@
+package com.shimmerresearch.verisense.payloaddesign;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.Arrays;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.shimmerresearch.driver.Configuration.COMMUNICATION_TYPE;
+import com.shimmerresearch.driverUtilities.ShimmerVerDetails.HW_ID;
+import com.shimmerresearch.sensors.AbstractSensor.SENSORS;
+import com.shimmerresearch.verisense.VerisenseDevice;
+import com.shimmerresearch.verisense.payloaddesign.DataBlockDetails.DATABLOCK_SENSOR_ID;
+import com.shimmerresearch.verisense.sensors.SensorMLX90632;
+import com.shimmerresearch.verisense.sensors.SensorVD6283;
+import com.shimmerresearch.verisense.sensors.SensorVD6283.VD6283_RATE;
+
+/**
+ * Tests for deriving the slow sensors' CSV gap-splitting window from the rate
+ * the PAYLOAD HEADER states, rather than measuring it from the data.
+ *
+ * The window is built by
+ * {@link UtilCsvSplitting#seedSlowSensorGapWindow(VerisenseDevice, DATABLOCK_SENSOR_ID)}
+ * and every split decision below is taken through the real
+ * {@link UtilCsvSplitting#isDataBlockContinuous(SENSORS, DataSegmentDetails, DataBlockDetails)},
+ * on synthetic payloads built the way the metadata parse leaves them. No binary
+ * test files, no hardware data and no reflection.
+ *
+ * Background. The VD6283 is sampled on a plain repeated timer at one of
+ * {@code 0.5, 1, 2, 5, 10, 20} Hz (hal_slowSensorSampler.c
+ * {@code slowSensorRateMs[]}), defaulting to 1 Hz, and 10 samples are buffered
+ * per block. That rate lived only in operational-config byte 75 and was not
+ * stored in the payload, so the parser fell back to the exposure-derived value,
+ * which bounds the rate from ABOVE and is ten times too fast at the default. A
+ * 10-sample block covering 10 s was laid out over 0.9 s and the remaining 9.1 s
+ * read as a gap: 129 one-block CSVs from one recording (DEV-979). From FW
+ * v2.02.000 the firmware stores the effective rate index in header byte 30 bits
+ * 6:3 (DEV-1011) and this suite pins the parser side of that.
+ *
+ * Several cases below are physical scenarios carried over from the earlier
+ * measurement-based work so the coverage is not lost: a dropped block, a failed
+ * I2C read costing one period, the VD6283's bimodal cadence, and an overlapping
+ * boundary. They are re-pointed at the header-derived window. The reason a
+ * measured window was abandoned is asserted directly by
+ * {@link #test006_droppedBlockSplitsOnTheVeryFirstBoundary()}: a window built
+ * from one or two observed gaps could not report a dropped block that was
+ * itself one of those gaps (DEV-974 Bug A), whereas a configured-rate window
+ * reports it from the first boundary onwards.
+ */
+public class API_00011_VerisenseSlowSensorGapWindow {
+
+ private static final int LIGHT_SAMPLES_PER_BLOCK = SensorVD6283.NUM_SAMPLES_PER_BLOCK;
+ private static final int SKIN_TEMP_SAMPLES_PER_BLOCK = SensorMLX90632.NUM_SAMPLES_PER_BLOCK;
+
+ private static final double TICKS_PER_SECOND = AsmBinaryFileConstants.TICKS_PER_SECOND;
+ private static final long TICKS_PER_MINUTE = (long) AsmBinaryFileConstants.TICKS_PER_MINUTE;
+
+ /** Skin temp refresh code 6 = 32 Hz refresh -> 16 Hz medical output (DEV-927). */
+ private static final int SKIN_TEMP_CONFIG_32HZ_REFRESH = 6<<1;
+ /** Refresh code 0 = 0.5 Hz refresh -> 0.25 Hz medical output (slowest). */
+ private static final int SKIN_TEMP_CONFIG_0HZ5_REFRESH = 0<<1;
+
+ @Before
+ public void clearSplittingState() {
+ UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor();
+ }
+
+ /**
+ * A gen-2 device as the payload-header parse leaves it. The config array is
+ * the 32-byte payload header, so writing a byte here is exactly what the
+ * firmware writing that byte would produce.
+ *
+ * @param lightGainAndDarkByte header byte 30: gain 2:0, rate index 6:3, dark 7
+ * @param skinTempConfigByte header byte 32: measType bit 0, refresh code 3:1
+ */
+ private VerisenseDevice setupGen2Device(int lightGainAndDarkByte, int skinTempConfigByte) {
+ return setupGen2Device(lightGainAndDarkByte, skinTempConfigByte, 2, 0, 9);
+ }
+
+ private VerisenseDevice setupGen2Device(int lightGainAndDarkByte, int skinTempConfigByte,
+ int fwMajor, int fwMinor, int fwInternal) {
+ VerisenseDevice device = new VerisenseDevice(COMMUNICATION_TYPE.SD);
+
+ byte[] configBytes = new byte[32];
+ configBytes[0] = (byte) 0x10; // extended-config flag
+ configBytes[2] = (byte) fwMajor;
+ configBytes[3] = (byte) fwMinor;
+ configBytes[4] = (byte) (fwInternal & 0xFF);
+ configBytes[5] = (byte) ((fwInternal>>8) & 0xFF);
+ configBytes[6] = (byte) 0xFF; // reset reason
+ configBytes[11] = HW_ID.VERISENSE_PULSE_PLUS; // SR68
+ configBytes[12] = 9; // SR68-9 (second generation)
+ configBytes[25] = (byte) (0x02 | (1<<3) | (1<<4)); // GEN_CFG_3: LED + VD6283 + MLX90632
+ configBytes[26] = (byte) lightGainAndDarkByte;
+ configBytes[28] = (byte) skinTempConfigByte;
+ device.configBytesParse(configBytes, COMMUNICATION_TYPE.SD);
+
+ device.getOrCreateListOfSensorClassKeysForDataBlockId(DATABLOCK_SENSOR_ID.LIGHT);
+ device.getOrCreateListOfSensorClassKeysForDataBlockId(DATABLOCK_SENSOR_ID.SKIN_TEMP);
+ assertTrue("this fixture must exercise the v11+ (uC ticks) path", device.isPayloadDesignV11orAbove());
+ return device;
+ }
+
+ /** Header byte 30 as the firmware writes it: gain 2.5x, given rate index, no dark channel. */
+ private static int lightHeaderByte(int rateIndex) {
+ return 0x02 | (rateIndex << SensorVD6283.LIGHT_RATE_INDEX_BIT_SHIFT);
+ }
+
+ /** A light device at the given configured rate index (0 = as earlier firmware left it). */
+ private VerisenseDevice setupLightDevice(int rateIndex) {
+ return setupGen2Device(lightHeaderByte(rateIndex), SKIN_TEMP_CONFIG_32HZ_REFRESH);
+ }
+
+ private static int samplesPerBlock(DATABLOCK_SENSOR_ID slowSensorId) {
+ return slowSensorId==DATABLOCK_SENSOR_ID.LIGHT? LIGHT_SAMPLES_PER_BLOCK:SKIN_TEMP_SAMPLES_PER_BLOCK;
+ }
+
+ private static SENSORS sensorClassKeyOf(DATABLOCK_SENSOR_ID slowSensorId) {
+ return slowSensorId==DATABLOCK_SENSOR_ID.LIGHT? SENSORS.VD6283:SENSORS.MLX90632;
+ }
+
+ /** A block as the metadata parse leaves it: sized, timed with the header rate, end ticks set. */
+ private DataBlockDetails newBlock(VerisenseDevice device, DATABLOCK_SENSOR_ID slowSensorId, long endTicks) {
+ int bytesPerSample = slowSensorId==DATABLOCK_SENSOR_ID.LIGHT? SensorVD6283.BYTES_PER_SAMPLE:SensorMLX90632.BYTES_PER_SAMPLE;
+ DataBlockDetails dataBlockDetails = new DataBlockDetails(slowSensorId, 0, 0,
+ device.getOrCreateListOfSensorClassKeysForDataBlockId(slowSensorId), 0, 0);
+ dataBlockDetails.setMetadata(samplesPerBlock(slowSensorId)*bytesPerSample, bytesPerSample,
+ device.getSamplingRateForSensor(sensorClassKeyOf(slowSensorId)));
+ // v11+ stores a SUB-MINUTE microcontroller tick counter per block...
+ dataBlockDetails.getTimeDetailsUcClock().setEndTimeTicks(endTicks%TICKS_PER_MINUTE);
+ // ...while the continuity check works on absolute real-world-clock ms.
+ dataBlockDetails.getTimeDetailsRwc().setEndTimeMs(endTicks/TICKS_PER_SECOND*1000);
+ return dataBlockDetails;
+ }
+
+ private static long ticks(double seconds) {
+ return (long) Math.round(seconds*TICKS_PER_SECOND);
+ }
+
+ /** Run the parse-flow step under test: derive the window from the header. */
+ private void seedWindow(VerisenseDevice device, DATABLOCK_SENSOR_ID slowSensorId) {
+ UtilCsvSplitting.seedSlowSensorGapWindow(device, slowSensorId);
+ }
+
+ private String continuityResult(DATABLOCK_SENSOR_ID slowSensorId, DataSegmentDetails previousSegment, DataBlockDetails next) {
+ return UtilCsvSplitting.isDataBlockContinuous(sensorClassKeyOf(slowSensorId), previousSegment, next);
+ }
+
+ private DataSegmentDetails dataSegmentOf(DataBlockDetails... dataBlockDetails) {
+ DataSegmentDetails dataSegmentDetails = new DataSegmentDetails();
+ for (DataBlockDetails block : dataBlockDetails) {
+ dataSegmentDetails.addDataBlock(block);
+ }
+ return dataSegmentDetails;
+ }
+
+ /**
+ * Walk a stream of one-block payloads through the real continuity check,
+ * asserting every boundary is judged continuous.
+ *
+ * @param spacingsS the spacing from each block to the next, in seconds
+ */
+ private DataSegmentDetails walkStream(VerisenseDevice device, DATABLOCK_SENSOR_ID slowSensorId, double firstBlockEndS, double... spacingsS) {
+ seedWindow(device, slowSensorId);
+ double endS = firstBlockEndS;
+ DataSegmentDetails dataSegmentDetails = dataSegmentOf(newBlock(device, slowSensorId, ticks(endS)));
+ for (int i = 0; i < spacingsS.length; i++) {
+ endS += spacingsS[i];
+ DataBlockDetails next = newBlock(device, slowSensorId, ticks(endS));
+ assertEquals("boundary " + i + " (spacing " + spacingsS[i] + " s) must be continuous",
+ "", continuityResult(slowSensorId, dataSegmentDetails, next));
+ dataSegmentDetails.addDataBlock(next);
+ }
+ return dataSegmentDetails;
+ }
+
+ private static double[] uniformSpacings(int count, double spacingS) {
+ double[] spacingsS = new double[count];
+ Arrays.fill(spacingsS, spacingS);
+ return spacingsS;
+ }
+
+ private double[] windowFor(DATABLOCK_SENSOR_ID slowSensorId) {
+ return UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.get(sensorClassKeyOf(slowSensorId));
+ }
+
+ // ------------------------------------------------- header rate decoding
+
+ /** Every index in the firmware rate table must decode to its rate. */
+ @Test
+ public void test001_everyConfiguredRateIndexIsDecoded() {
+ // hal_slowSensorSampler.c: slowSensorRateMs[] = {0, 2000, 1000, 500, 200, 100, 50}.
+ // The table is a STORED-DATA CONTRACT shared with the firmware across two
+ // repositories with no build-time link between them, so pin the conversion
+ // here: an index renumbered on either side silently re-times recordings.
+ double[] firmwarePeriodMs = {2000, 1000, 500, 200, 100, 50};
+ double[] expectedHz = {0.5, 1.0, 2.0, 5.0, 10.0, 20.0};
+ for(int i=0;i
+ * The obvious repair is to narrow the slow side to {@code cfg/1.5}, as the
+ * VD6283 uses. That was tried and reverted: Test_065 contains a HEALTHY
+ * skin-temp boundary at 1.63x nominal spacing during start-up settling, and
+ * narrowing split it. This sensor's healthy behaviour genuinely reaches into
+ * the region a dropped block occupies, so the two assertions below are the
+ * real constraint from both sides, 1.63x must not split and 1.75x must.
+ */
+ @Test
+ public void test017_skinTempGapEdgeSeparatesRealLossFromStartUpSettling() {
+ VerisenseDevice device = setupGen2Device(lightHeaderByte(2), SKIN_TEMP_CONFIG_32HZ_REFRESH);
+ seedWindow(device, DATABLOCK_SENSOR_ID.SKIN_TEMP);
+ double[] window = windowFor(DATABLOCK_SENSOR_ID.SKIN_TEMP);
+
+ double configuredHz = 16.0;
+
+ // Measured on Test_065: a 16-sample block boundary 1531 ms after the
+ // previous block ended, against a nominal 937.5 ms. No samples were lost.
+ assertFalse("the 1.63x start-up boundary observed on Test_065 must NOT split",
+ UtilCsvSplitting.isSamplingRateOutsideOfLimits(window, configuredHz/1.633));
+
+ // A dropped block, even hidden by a 12.5% catch-up, must still split.
+ assertTrue("a dropped block landing on a catch-up must split",
+ UtilCsvSplitting.isSamplingRateOutsideOfLimits(window, configuredHz/(2.0*0.875)));
+
+ // A clean dropped block has proper margin; it is only the catch-up case
+ // that is close.
+ assertTrue(UtilCsvSplitting.isSamplingRateOutsideOfLimits(window, configuredHz/2.0));
+ assertTrue("the catch-up case is deliberately close - if this margin ever grows"
+ + " past 10% somebody has widened the window without saying so",
+ (window[0]-configuredHz/(2.0*0.875))/window[0] < 0.10);
+ }
+
+ /**
+ * The slowest skin-temp configuration, 0.25 Hz output, where a 16-sample
+ * block spans about 64 s. Two such blocks can land in one payload, and
+ * differencing their SUB-MINUTE end ticks re-bases a real 64 s gap to about
+ * 4 s, i.e. an apparent 4 Hz. That is what fragmented a 3-day recording into
+ * thousands of CSVs and, through the algorithm-buffer reset on every split,
+ * stopped non-wear detection producing any output. Taking the rate from the
+ * header removes the tick arithmetic entirely.
+ */
+ @Test
+ public void test018_skinTempAtSlowestRateUsesTheHeaderRateNotWrappedTicks() {
+ VerisenseDevice device = setupGen2Device(lightHeaderByte(2), SKIN_TEMP_CONFIG_0HZ5_REFRESH);
+ assertEquals("slowest configuration is 0.25 Hz output", 0.25, device.getSamplingRateForSensor(SENSORS.MLX90632), 1e-9);
+ seedWindow(device, DATABLOCK_SENSOR_ID.SKIN_TEMP);
+
+ double[] window = windowFor(DATABLOCK_SENSOR_ID.SKIN_TEMP);
+ assertNotNull(window);
+ assertTrue("the window must sit around 0.25 Hz, not the wrapped ~4 Hz", window[1] < 1.0);
+
+ // A genuine 64 s boundary between 0.25 Hz blocks must NOT split.
+ walkStream(device, DATABLOCK_SENSOR_ID.SKIN_TEMP, 10, 64.0, 64.0, 64.0);
+ // The aliased ~4 Hz reading that the tick path produced would split.
+ assertTrue(UtilCsvSplitting.isSamplingRateOutsideOfLimits(window, 4.0));
+ }
+
+ // ------------------------------------------------------------- other
+
+ // ------------------------------------------- the firmware-fault diagnostic
+
+ /**
+ * Captures whatever the parser prints while {@code runnable} runs.
+ *
+ * Worth the awkwardness: the warning below is the ONLY signal that would
+ * exist if the firmware shipped with the rate field broken, so "does it
+ * actually print" is the whole point of it.
+ */
+ private String captureConsole(Runnable runnable) {
+ PrintStream original = System.out;
+ ByteArrayOutputStream captured = new ByteArrayOutputStream();
+ try {
+ System.setOut(new PrintStream(captured));
+ runnable.run();
+ } finally {
+ System.setOut(original);
+ }
+ return captured.toString();
+ }
+
+ /**
+ * A recording from firmware that predates the field is the ordinary case and
+ * must stay silent. Warning on every old recording would train people to
+ * ignore the line that matters.
+ */
+ @Test
+ public void test023_oldFirmwareWithNoStoredRateIsNotReported() {
+ final VerisenseDevice device = setupGen2Device(lightHeaderByte(0), SKIN_TEMP_CONFIG_32HZ_REFRESH, 2, 0, 9);
+ assertFalse("fixture must have no stored rate", device.getSensorVD6283().isConfiguredRateKnown());
+ assertFalse("and must predate the field", device.isPayloadDesignV14orAbove());
+
+ String printed = captureConsole(new Runnable() {
+ @Override
+ public void run() {
+ UtilCsvSplitting.warnIfLightRateFieldUnusable(device);
+ }
+ });
+ assertEquals("an old recording must print nothing", "", printed.trim());
+ }
+
+ /**
+ * The case this exists for: firmware new enough to store the rate, carrying
+ * light blocks, with the field clear. That is a firmware fault, and without
+ * this line it would be indistinguishable from an old recording - the file
+ * would parse, the light data would come out, and the near-blind fallback
+ * window would never be mentioned.
+ */
+ @Test
+ public void test024_newFirmwareWithNoStoredRateIsReportedOncePerFile() {
+ final VerisenseDevice device = setupGen2Device(lightHeaderByte(0), SKIN_TEMP_CONFIG_32HZ_REFRESH, 2, 2, 0);
+ assertTrue("fixture must claim firmware that stores the field", device.isPayloadDesignV14orAbove());
+ assertFalse("but carry no rate", device.getSensorVD6283().isConfiguredRateKnown());
+
+ String printed = captureConsole(new Runnable() {
+ @Override
+ public void run() {
+ // Three payloads of the same file: the warning is per recording.
+ UtilCsvSplitting.warnIfLightRateFieldUnusable(device);
+ UtilCsvSplitting.warnIfLightRateFieldUnusable(device);
+ UtilCsvSplitting.warnIfLightRateFieldUnusable(device);
+ }
+ });
+ assertTrue("it must say the firmware is at fault: " + printed, printed.contains("firmware fault"));
+ assertEquals("exactly one line per file", 1, printed.split("WARNING", -1).length-1);
+
+ // A new file clears the state, so the next recording is reported too.
+ UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor();
+ String nextFile = captureConsole(new Runnable() {
+ @Override
+ public void run() {
+ UtilCsvSplitting.warnIfLightRateFieldUnusable(device);
+ }
+ });
+ assertTrue("the next file must warn again", nextFile.contains("WARNING"));
+ }
+
+ /** New firmware that DID store the rate is the healthy case: silent. */
+ @Test
+ public void test025_newFirmwareWithAStoredRateIsNotReported() {
+ final VerisenseDevice device = setupGen2Device(lightHeaderByte(2), SKIN_TEMP_CONFIG_32HZ_REFRESH, 2, 2, 0);
+ assertTrue(device.getSensorVD6283().isConfiguredRateKnown());
+
+ String printed = captureConsole(new Runnable() {
+ @Override
+ public void run() {
+ UtilCsvSplitting.warnIfLightRateFieldUnusable(device);
+ }
+ });
+ assertEquals("", printed.trim());
+ }
+
+ /**
+ * A reserved index is reported whatever the payload design says.
+ *
+ * Two reasons it cannot be folded into the previous case. It is a different
+ * fault, so saying the field is "clear" would send a reader after the wrong
+ * firmware bug. And on an OLD recording those bits should be clear by
+ * construction, so a non-zero one is an anomaly in its own right - gating the
+ * report on the firmware version would have said nothing at all.
+ */
+ @Test
+ public void test026_reservedRateIndexIsReportedWhateverTheFirmwareVersion() {
+ for(final int[] fw:new int[][] {{2, 0, 9}, {2, 2, 0}}) {
+ UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor();
+ final VerisenseDevice device = setupGen2Device(
+ lightHeaderByte(9), SKIN_TEMP_CONFIG_32HZ_REFRESH, fw[0], fw[1], fw[2]);
+ assertTrue(device.getSensorVD6283().isRateIndexReserved());
+
+ String printed = captureConsole(new Runnable() {
+ @Override
+ public void run() {
+ UtilCsvSplitting.warnIfLightRateFieldUnusable(device);
+ }
+ });
+ String where = "firmware " + fw[0] + "." + fw[1] + "." + fw[2] + ": " + printed;
+ assertTrue("a reserved index must be reported, " + where, printed.contains("WARNING"));
+ assertTrue("and named, " + where, printed.contains("reserved index 9"));
+ assertFalse("it must not be described as clear, " + where, printed.contains("field clear"));
+ }
+ }
+
+ // ------------------------------------- the layer above the window seeder
+
+ /**
+ * Everything else here drives {@link UtilCsvSplitting} directly. This drives
+ * the layer that decides WHETHER the seeder runs during a real payload parse.
+ *
+ * That layer matters for a reason that is invisible from the seeder itself:
+ * asking the device for the sensor-class keys behind a data block id CREATES
+ * and caches them, and the lookup returns the second-generation slow sensors
+ * unconditionally. Seeding for a sensor the payload has no blocks for would
+ * therefore populate mappings and rate limits for hardware the recording does
+ * not have - visible on first-generation files, which have neither slow sensor.
+ */
+ @Test
+ public void test027_windowIsSeededOnlyForSensorsThePayloadActuallyCarries() {
+ VerisenseDevice device = setupLightDevice(2);
+ PayloadContentsDetailsV8orAbove payload = new PayloadContentsDetailsV8orAbove(device);
+
+ assertFalse("an empty payload carries neither slow sensor",
+ payload.containsDataBlockForSensor(DATABLOCK_SENSOR_ID.LIGHT));
+ payload.seedSlowSensorGapWindow(DATABLOCK_SENSOR_ID.LIGHT);
+ assertNull("no blocks means no window", windowFor(DATABLOCK_SENSOR_ID.LIGHT));
+ assertNull(windowFor(DATABLOCK_SENSOR_ID.SKIN_TEMP));
+
+ // Give it a light block and the light window appears - and only that one.
+ payload.listOfDataBlocksInOrder.add(newBlock(device, DATABLOCK_SENSOR_ID.LIGHT, ticks(10)));
+ assertTrue(payload.containsDataBlockForSensor(DATABLOCK_SENSOR_ID.LIGHT));
+ assertFalse(payload.containsDataBlockForSensor(DATABLOCK_SENSOR_ID.SKIN_TEMP));
+
+ payload.seedSlowSensorGapWindow(DATABLOCK_SENSOR_ID.LIGHT);
+ payload.seedSlowSensorGapWindow(DATABLOCK_SENSOR_ID.SKIN_TEMP);
+
+ double[] lightWindow = windowFor(DATABLOCK_SENSOR_ID.LIGHT);
+ assertNotNull("a payload carrying light blocks must get its window", lightWindow);
+ assertEquals(1.0/UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER.SLOW_SENSOR_MAX_INTER_BLOCK_GAP_RATIO,
+ lightWindow[0], 1e-9);
+ assertNull("and the sensor with no blocks must get nothing",
+ windowFor(DATABLOCK_SENSOR_ID.SKIN_TEMP));
+ }
+
+ /** Nothing here may disturb a fast sensor's own band. */
+ @Test
+ public void test019_fastSensorLimitsAreUntouched() {
+ VerisenseDevice device = setupLightDevice(2);
+ double[] fastSensorLimits = UtilCsvSplitting.calculateSamplingRateLimits(960);
+ UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.put(SENSORS.LSM6DSV, fastSensorLimits);
+
+ seedWindow(device, DATABLOCK_SENSOR_ID.LIGHT);
+
+ assertTrue("the fast sensor's band must be the same array, unmodified",
+ fastSensorLimits==UtilCsvSplitting.SAMPLING_RATE_LIMITS_PER_SENSOR.get(SENSORS.LSM6DSV));
+ assertEquals(960*UtilCsvSplitting.FILE_GAP_TOLERANCE_MULTIPLIER.LOWER, fastSensorLimits[0], 1e-9);
+ }
+
+ /**
+ * The window is a pure function of the header, so re-seeding it - which
+ * happens on every payload - always yields the same band, and clearing the
+ * state at a CSV-set boundary loses nothing that cannot be recomputed.
+ */
+ @Test
+ public void test020_windowIsStatelessAndReproducible() {
+ VerisenseDevice device = setupLightDevice(2);
+ seedWindow(device, DATABLOCK_SENSOR_ID.LIGHT);
+ double[] first = windowFor(DATABLOCK_SENSOR_ID.LIGHT).clone();
+
+ for (int i = 0; i < 50; i++) {
+ seedWindow(device, DATABLOCK_SENSOR_ID.LIGHT);
+ }
+ assertArrayEqualsExact(first, windowFor(DATABLOCK_SENSOR_ID.LIGHT));
+
+ UtilCsvSplitting.clearMapOfSamplingRateLimitsPerSensor();
+ seedWindow(device, DATABLOCK_SENSOR_ID.LIGHT);
+ assertArrayEqualsExact(first, windowFor(DATABLOCK_SENSOR_ID.LIGHT));
+ }
+
+ private static void assertArrayEqualsExact(double[] expected, double[] actual) {
+ assertNotNull(actual);
+ assertEquals(expected.length, actual.length);
+ for (int i = 0; i < expected.length; i++) {
+ assertEquals(expected[i], actual[i], 0.0);
+ }
+ }
+
+ /**
+ * Pins the regression this change exists to fix, without needing the old
+ * implementation to hand.
+ *
+ * Before the rate was stored, the light window could only come from
+ * {@code getRateFreq()}, which was the exposure bound - 10 Hz at the default
+ * 100 ms exposure - widened by the standard +/-10%. A 1 Hz recording presents
+ * 10 samples per 10 s, i.e. 1 Hz, which sits far outside that band, so EVERY
+ * boundary was reported as a gap and every block became its own CSV: 129 of
+ * them from one 22-minute recording (DEV-979). The header-derived window
+ * accepts the same boundary.
+ */
+ @Test
+ public void test021_theExposureDerivedBandIsWhatUsedToSplitEveryBlock() {
+ double exposureBoundHz = setupLightDevice(0).getSensorVD6283().getRateFreq();
+ assertEquals("the old band was built from this", 10.0, exposureBoundHz, 1e-9);
+ double[] oldBand = UtilCsvSplitting.calculateSamplingRateLimits(exposureBoundHz);
+
+ double apparentRateOf1HzBoundaryHz = LIGHT_SAMPLES_PER_BLOCK/10.0;
+ assertTrue("a healthy 1 Hz boundary was OUTSIDE the exposure-derived band",
+ UtilCsvSplitting.isSamplingRateOutsideOfLimits(oldBand, apparentRateOf1HzBoundaryHz));
+
+ VerisenseDevice device = setupLightDevice(2);
+ seedWindow(device, DATABLOCK_SENSOR_ID.LIGHT);
+ assertFalse("and INSIDE the header-derived window",
+ UtilCsvSplitting.isSamplingRateOutsideOfLimits(windowFor(DATABLOCK_SENSOR_ID.LIGHT), apparentRateOf1HzBoundaryHz));
+ }
+
+ /**
+ * The CSV sensor-config line reports {@code Configured} only when the header
+ * actually carried the rate, so a reader can tell a known rate from an
+ * exposure-derived guess.
+ */
+ @Test
+ public void test022_csvConfigLineReportsConfiguredOnlyWhenKnown() {
+ String known = setupLightDevice(2).generateSensorConfigStrSingleSensor(SENSORS.VD6283, 0.993);
+ assertTrue("known rate must report Configured: " + known, known.contains("Configured = 1.0 Hz"));
+ assertTrue(known.contains("Calculated = 0.993 Hz"));
+ assertTrue("the rest of the line must be preserved: " + known,
+ known.contains("Gain = 2.50x") && known.contains("Slot1 = Visible"));
+
+ String unknown = setupLightDevice(0).generateSensorConfigStrSingleSensor(SENSORS.VD6283, 0.993);
+ assertFalse("an unknown rate must NOT be reported as configured: " + unknown,
+ unknown.contains("Configured"));
+ assertTrue(unknown.contains("Calculated = 0.993 Hz"));
+ assertTrue(unknown.contains("Gain = 2.50x") && unknown.contains("Slot1 = Visible"));
+ }
+
+ /**
+ * {@code Configured} is the rate the device was asked for, NOT the
+ * exposure-clamped one the parser times blocks with.
+ *
+ * test022 cannot see the difference, and neither can any other test here: at
+ * 1 Hz against the fixture exposure of 100 ms the clamped and unclamped values
+ * are the same number. 20 Hz is where they part, since the chip cannot measure
+ * faster than it integrates and the bound is 10 Hz. Someone reading the CSV to
+ * check what the device was configured to do has to see 20, while
+ * {@link SensorVD6283#getRateFreq()} goes on returning 10 for the block timing
+ * and the gap window - test005 pins that side.
+ */
+ @Test
+ public void test028_csvConfiguredValueIsTheRateAskedForNotTheClampedOne() {
+ VerisenseDevice device = setupLightDevice(6); // 20 Hz configured
+ assertEquals("the fixture must actually clamp, or this proves nothing",
+ 10.0, device.getSensorVD6283().getRateFreq(), 1e-9);
+
+ String line = device.generateSensorConfigStrSingleSensor(SENSORS.VD6283, 9.912);
+ assertTrue("Configured must report the configured rate: " + line,
+ line.contains("Configured = 20.0 Hz"));
+ assertFalse("and must not report the exposure clamp instead: " + line,
+ line.contains("Configured = 10.0 Hz"));
+ assertTrue(line.contains("Calculated = 9.912 Hz"));
+ }
+}