diff --git a/.github/spc-format-updated.md b/.github/spc-format-updated.md deleted file mode 100644 index e9c2adb..0000000 --- a/.github/spc-format-updated.md +++ /dev/null @@ -1,1180 +0,0 @@ -# GRAMS SPC File Format - -This document summarizes the GRAMS SPC binary format as exposed by Galactic's SPC SDK (SPC.H) and Universal Data File specification. It is aimed at developers implementing SPC readers/writers, with emphasis on single-spectrum and simple multifile XY data. - -> The canonical reference remains the original SPC SDK headers and UDF specification. This document is a practical overview, not a verbatim copy. - ---- - -## 1. High-level overview - -An SPC file is a binary container for one or more spectra ("subfiles"): - -1. **Main header**: fixed 512-byte structure (SPCHDR) in the "new" format. -2. **Optional global X array**: `fnpts` 32-bit floats if TXVALS is set and TXYXYS is clear. -3. **One or more subfiles**: - - SUBHDR (32-byte per subfile header). - - Optional per-subfile X array (when TXYXYS is set). - - Y data array (16-bit or 32-bit fixed-point or 32-bit IEEE float). -4. **Optional XY directory** for TXYXYS multifiles (array of SSFSTC). -5. **Optional log block** containing a LOGSTC header, optional binary data, and log text. - -There is also an older "0x4D" format with a shorter header (224 or 256 bytes) and some different field types. Modern software normally writes the new 512-byte header form. - -### File naming conventions - -By convention, SPC files use specific extensions to indicate data type: - -| Extension | Data Type | -|-----------|-----------| -| `.SPC` | Spectrum (general case) | -| `.CGM` | Chromatogram (general case) | -| `.GC` | Gas Chromatogram | -| `.LC` | Liquid Chromatogram | -| `.HC` | HPLC Chromatogram | -| `.FIR` | Infrared (FT-IR) Spectrum | -| `.IR` | Near Infrared (NIR) Spectrum | -| `.VIS` | Visible Spectrum | -| `.UV` | Ultraviolet (Visible) Spectrum | -| `.XRY` | X-ray Spectrum | -| `.MS` | Mass Spectrum | -| `.NMR` | Nuclear Magnetic Resonance Spectrum | -| `.RMN` | Raman Spectrum | -| `.EFL` | Fluorescence Spectrum | -| `.AAS` | Atomic Spectrum | -| `.DAS` | Diode Array Spectrum | - -For hyphenated techniques (e.g., GC-IR, GC-MS), GRAMS software automatically "links" files if spectral data is in a `.SPC` file and chromatogram data is in a `.CGM` file with the same base name in the same directory. - ---- - -## 2. Endianness and versions - -The main header includes a version/endianness byte `fversn`: - -- `0x4B`: new format, little-endian (LSB first). This is the common case. -- `0x4C`: new format, big-endian (MSB first). -- `0x4D`: old format (header 224/256 bytes, some fields differ, 32-bit Y words are word-swapped). - -For a Python parser using `struct`, you typically: - -- Read the 512-byte header using little-endian (`<`) and check that `fversn == 0x4B` for now. -- Optionally add support for `0x4D` later (see section 13). - -The new format supports multifiles, the Audit Log, and has a 512-byte main header. The old format does not support these features and uses a 256-byte main header. - ---- - -## 3. Basic C types and Python `struct` formats - -SPC structures use fixed-width C types with no padding between fields. Common types and their sizes: - -| C type | Size (bytes) | Python `struct` (little-endian) | -| ----------- | ------------ | -------------------------------- | -| `BYTE` | 1 | `B` | -| `char` | 1 | `c` or `s` (for arrays) | -| `WORD` | 2 (uint16) | `H` | -| `DWORD` | 4 (uint32) | `I` | -| `float` | 4 (IEEE-754) | `f` | -| `double` | 8 (IEEE-754) | `d` | -| `char[n]` | n | `ns` (e.g. `9s`, `130s`) | - -Use a leading `<` or `>` in the format string depending on endianness. - -Example for reading a `DWORD` followed by a `double` in Python: - -```python -npts, first_x = struct.unpack(' float Y - DWORD fnpts; // # of points, or XY directory offset for TXYXYS - double ffirst; // X of first point - double flast; // X of last point - DWORD fnsub; // # of subfiles (1 if not TMULTI) - BYTE fxtype; // X axis unit type - BYTE fytype; // Y axis unit type - BYTE fztype; // Z axis unit type - BYTE fpost; // posting disposition - DWORD fdate; // packed date/time - char fres[9]; // resolution string - char fsource[9]; // instrument string - WORD fpeakpt; // peak point for interferograms (0 if unknown) - float fspare[8]; // reserved / internal (used by Array Basic) - char fcmnt[130]; // comment text - char fcatxt[30]; // axis labels if TALABS - DWORD flogoff; // offset of log block, or 0 - DWORD fmods; // modification flags - BYTE fprocs; // processing code - BYTE flevel; // calibration level + 1 - WORD fsampin; // sample injection number - float ffactor; // multiplier / concentration factor - char fmethod[48]; // method / program / data filenames - float fzinc; // Z increment (0 => use first subfile delta) - DWORD fwplanes; // # of W-planes (4D data), or 0 - float fwinc; // W increment (if fwplanes != 0) - BYTE fwtype; // W axis unit type - char freserv[187]; // reserved (must be zero) -} SPCHDR; -``` - -Field summary: - -| Name | Type | `struct` code | Description | -| --------- | ----------- | ------------- | ----------------------------------- | -| `ftflgs` | `uint8` | `B` | Flags (precision, multifile, X mode) | -| `fversn` | `uint8` | `B` | Format / endianness (0x4B, 0x4C, 0x4D) | -| `fexper` | `uint8` | `B` | Instrument / technique code | -| `fexp` | `int8` | `b` | Y exponent (0x80 => float Y values) | -| `fnpts` | `uint32` | `I` | Point count (or XY dir offset TXYXYS) | -| `ffirst` | `double` | `d` | X coordinate of first point | -| `flast` | `double` | `d` | X coordinate of last point | -| `fnsub` | `uint32` | `I` | Number of subfiles | -| `fxtype` | `uint8` | `B` | X axis unit type | -| `fytype` | `uint8` | `B` | Y axis unit type | -| `fztype` | `uint8` | `B` | Z axis unit type | -| `fpost` | `uint8` | `B` | Posting disposition (see 4.2.1) | -| `fdate` | `uint32` | `I` | Packed date/time | -| `fres` | `char[9]` | `9s` | Resolution text | -| `fsource` | `char[9]` | `9s` | Source instrument text | -| `fpeakpt` | `uint16` | `H` | Peak point index (interferograms) | -| `fspare` | `float[8]` | `8f` | Reserved / internal | -| `fcmnt` | `char[130]` | `130s` | Comment text | -| `fcatxt` | `char[30]` | `30s` | Axis label strings (if TALABS) | -| `flogoff` | `uint32` | `I` | Offset of log block | -| `fmods` | `uint32` | `I` | Modification flags | -| `fprocs` | `uint8` | `B` | Processing code | -| `flevel` | `uint8` | `B` | Calibration level + 1 | -| `fsampin` | `uint16` | `H` | Sample injection number | -| `ffactor` | `float` | `f` | Data multiplier / concentration | -| `fmethod` | `char[48]` | `48s` | Method/program filenames | -| `fzinc` | `float` | `f` | Z increment | -| `fwplanes`| `uint32` | `I` | Number of W-planes (4D) | -| `fwinc` | `float` | `f` | W increment | -| `fwtype` | `uint8` | `B` | W axis unit type | -| `freserv` | `char[187]` | `187s` | Reserved (zeros) | - -### 4.1 Python `struct` format for SPCHDR (little-endian) - -A minimal format string for the fields above in Python (omitting any padding) is: - -```python -SPCHDR_FMT = ' **Note for readers**: This field was primarily used by GRAMS software for workflow control and does not affect how spectral data is parsed or interpreted. Modern readers can treat it as metadata only. - -#### 4.2.2 Packed date format (`fdate`) - -The date/time is encoded as unsigned integers into a 32-bit value (most significant bit on left): - -- **Year**: 12 bits (0-4095) -- **Month**: 4 bits (1-12) -- **Day**: 5 bits (1-31) -- **Hour**: 5 bits (0-23) -- **Minute**: 6 bits (0-59) - -Example Python decoding: - -```python -minute = fdate & 0x3F -hour = (fdate >> 6) & 0x1F -day = (fdate >> 11) & 0x1F -month = (fdate >> 16) & 0x0F -year = (fdate >> 20) & 0xFFF -``` - -### 4.3 Axis unit types (X, Y, Z, W) - -The headers define enumerations for axis types. Common values: - -**X axis** (`fxtype`, `fztype`, `fwtype`): - -| Code | Meaning | -| ---- | ----------------------- | -| 0 | Arbitrary | -| 1 | Wavenumber (cm⁻¹) | -| 2 | Micrometers | -| 3 | Nanometers | -| 4 | Seconds | -| 5 | Minutes | -| 30 | Hours | -| 6 | Hertz (Hz) | -| 7 | Kilohertz (kHz) | -| 8 | Megahertz (MHz) | -| 26 | Gigahertz (GHz) | -| 9 | Mass (m/z) | -| 13 | Raman shift (cm⁻¹) | -| 22 | Data points | -| 18 | Degrees | -| 19 | Temperature (F) | -| 20 | Temperature (C) | -| 21 | Temperature (K) | -| 27 | Centimeters (cm) | -| 28 | Meters (m) | -| 29 | Millimeters (mm) | -| 23 | Milliseconds (mSec) | -| 24 | Microseconds (uSec) | -| 25 | Nanoseconds (nSec) | - -**Y axis** (`fytype`): - -| Code | Meaning | -| ---- | ----------------------- | -| 0 | Arbitrary intensity | -| 1 | Interferogram | -| 2 | Absorbance | -| 3 | Kubelka-Munk | -| 10 | Log(1/R) | -| 11 | Percent | -| 4 | Counts | -| 5 | Volts | -| 9 | Millivolts (mV) | -| 12 | Intensity | -| 13 | Relative intensity | -| 128 | Transmission (valleys) | -| 129 | Reflectance | -| 130 | Arbitrary or Single Beam with Valley Peaks | -| 131 | Emission | - -**Note**: All Y types ≥128 are assumed to have inverted (valley) peaks. - -For full lists and rare values, see the original header definitions. - -#### Custom axis labels - -If the `TALABS` flag is set in `ftflgs`, axis labels are taken from the `fcatxt` field instead of the enumerated types. The `fcatxt` contains three null-terminated strings for X, Y, and Z labels in that order. Each label can be up to 20 characters, and all three must fit within 30 bytes total. - -If a label in `fcatxt` is just a null byte (empty string), the corresponding enumerated type label is used instead. - ---- - -## 5. Subfile header (SUBHDR) - -Every file, even single-spectrum files, has at least one subfile header immediately before the Y data for that subfile. In C: - -```c -typedef struct { - BYTE subflgs; // changed / no-peak-table / modified flags - char subexp; // Y exponent for this subfile (0x80 => float) - WORD subindx; // subfile index (0 = first) - float subtime; // Z coordinate (time, etc.) for this subfile - float subnext; // Z coordinate for next subfile - float subnois; // noise estimate (high byte non-zero if valid) - DWORD subnpts; // # points in TXYXYS mode; ignored otherwise - DWORD subscan; // # co-added scans or 0 - float subwlevel; // W-axis value for this subfile (4D) - char subresv[4];// reserved (zero) -} SUBHDR; -``` - -Field summary: - -| Name | Type | `struct` code | Description | -| ----------- | ----------- | ------------- | ------------------------------------ | -| `subflgs` | `uint8` | `B` | Subfile flags (changed, etc.) | -| `subexp` | `int8` | `b` | Y exponent for this subfile | -| `subindx` | `uint16` | `H` | Subfile index (0 = first) | -| `subtime` | `float` | `f` | Z coordinate (time, etc.) | -| `subnext` | `float` | `f` | Z coordinate of next subfile | -| `subnois` | `float` | `f` | Noise estimate | -| `subnpts` | `uint32` | `I` | Point count in TXYXYS mode | -| `subscan` | `uint32` | `I` | Number of co-added scans | -| `subwlevel` | `float` | `f` | W-axis value for this subfile | -| `subresv` | `char[4]` | `4s` | Reserved (zeros) | - -Python `struct` format (little-endian): - -```python -SUBHDR_FMT = '= logsizd) - DWORD logtxto; // offset from start of LOGSTC to text - DWORD logbins; // size of binary area after LOGSTC - DWORD logdsks; // size of disk-only area after binary - char logspar[44]; // reserved (zero) -} LOGSTC; -``` - -Field summary: - -| Name | Type | `struct` code | Description | -| ---------- | ----------- | ------------- | ---------------------------------------- | -| `logsizd` | `uint32` | `I` | Total size of disk log block (bytes) | -| `logsizm` | `uint32` | `I` | Size of in-memory block (>= logsizd) | -| `logtxto` | `uint32` | `I` | Offset from LOGSTC start to log text | -| `logbins` | `uint32` | `I` | Size of binary area after LOGSTC | -| `logdsks` | `uint32` | `I` | Size of disk-only area after binary | -| `logspar` | `char[44]` | `44s` | Reserved (zeros) | - -Python format (little-endian): - -```python -LOGSTC_FMT = ' bytes: + raw = text.encode("latin-1", errors="replace") + raw = raw[:length] + return raw + b"\x00" * (length - len(raw)) + + +def _pack_date_int(year: int, month: int, day: int, hour: int, minute: int) -> int: + # Packed: YYYY(12) MM(4) DD(5) HH(5) MM(6) + return (year << 20) | (month << 16) | (day << 11) | (hour << 6) | minute + + +def _pack_header_le(fields: dict[str, object]) -> bytes: + buf = bytearray() + for name, fmt in SPC_HEADER_FIELDS: + value = fields[name] + if fmt.endswith("s"): + size = int(fmt[:-1]) + assert isinstance(value, (bytes, bytearray)) + if len(value) != size: + raise ValueError(f"Field {name} must be {size} bytes, got {len(value)}") + buf += value + elif fmt.endswith("f") and fmt != "f": + # e.g. 8f + assert isinstance(value, Iterable) + buf += struct.pack("<" + fmt, *value) + else: + buf += struct.pack("<" + fmt, value) + + if len(buf) != SPC_HEADER_SIZE: + raise ValueError(f"Header size mismatch: got {len(buf)} bytes, expected {SPC_HEADER_SIZE}") + return bytes(buf) + + +def _pack_subheader_le(fields: dict[str, object]) -> bytes: + buf = bytearray() + for name, fmt in SPC_SUBHEADER_FIELDS: + value = fields[name] + if fmt.endswith("s"): + size = int(fmt[:-1]) + assert isinstance(value, (bytes, bytearray)) + if len(value) != size: + raise ValueError(f"Subheader field {name} must be {size} bytes, got {len(value)}") + buf += value + else: + buf += struct.pack("<" + fmt, value) + + if len(buf) != SPC_SUBHEADER_SIZE: + raise ValueError(f"Subheader size mismatch: got {len(buf)} bytes, expected {SPC_SUBHEADER_SIZE}") + return bytes(buf) + + +def _encode_y_fixed_int32(y_float: np.ndarray, exponent: int) -> bytes: + # Reader decodes: y = int32 * (2**exponent / 2**32) + # So int32 = y * (2**32 / 2**exponent) + scale = (2.0**32) / (2.0**exponent) + y_raw = np.clip(np.round(y_float * scale), np.iinfo(np.int32).min, np.iinfo(np.int32).max).astype(" bytes: + # Reader decodes: y = int16 * (2**exponent / 2**16) + scale = (2.0**16) / (2.0**exponent) + y_raw = np.clip(np.round(y_float * scale), np.iinfo(np.int16).min, np.iinfo(np.int16).max).astype(" dict[str, object]: + return { + "flags": flags, + "version": 0x4B, + "experiment_type": 0, + "exponent": 0, + "n_points": int(n_points), + "first_x": 0.0, + "last_x": 1.0, + "n_subfiles": int(n_subfiles), + "x_unit_code": 0, + "y_unit_code": 0, + "z_unit_code": 0, + "posting_disposition": 0, + "date_int": 0, + "resolution_str": _cstr("", 9), + "source_str": _cstr("", 9), + "peak_point_index": 0, + "spare_floats": (0.0,) * 8, + "comment": _cstr("", 130), + "axis_label_text": _cstr("", 30), + "log_offset": 0, + "modification_flags": 0, + "processing_code": 0, + "calibration_level_raw": 0, + "sample_injection_number": 0, + "data_multiplier": 0.0, + "method_text": _cstr("", 48), + "z_increment": 0.0, + "w_planes": 0, + "w_increment": 0.0, + "w_unit_code": 0, + "reserved": b"\x00" * 187, + } + + +def make_shared_x_single_evenx() -> bytes: + n_points = 1844 + header = _default_header(flags=0, n_points=n_points, n_subfiles=1) + header["first_x"] = 447.48 + header["last_x"] = 4002.28 + header["x_unit_code"] = 1 # Wavenumber + header["y_unit_code"] = 2 # Absorbance + + rng = np.random.default_rng(123) + y = rng.normal(loc=0.0, scale=0.2, size=n_points).astype(np.float64) + + subhdr = { + "flags": 0, + "exponent": 0, + "subfile_index": 0, + "z_value": 0.0, + "z_next": 0.0, + "noise": 0.0, + "n_points": 0, + "n_scans": 0, + "w_value": 0.0, + "reserved": b"\x00" * 4, + } + + return b"".join([ + _pack_header_le(header), + _pack_subheader_le(subhdr), + _encode_y_fixed_int32(y, exponent=int(header["exponent"])), + ]) + + +def make_shared_x_single_explicit_x() -> bytes: + n_points = 256 + flags = FLAG_EXPLICIT_X + + header = _default_header(flags=flags, n_points=n_points, n_subfiles=1) + header["first_x"] = 0.0 + header["last_x"] = 10.0 + header["x_unit_code"] = 5 # Time (min) + header["y_unit_code"] = 0 # Arbitrary Intensity + header["date_int"] = _pack_date_int(1986, 1, 9, 8, 47) + + x_lin = np.linspace(header["first_x"], header["last_x"], n_points, dtype=np.float64) + # Make it monotonic but not equal to linspace at rtol=1e-6. + x = (x_lin + (np.arange(n_points, dtype=np.float64) / n_points) * 1e-3).astype(" bytes: + n_points = 171 + n_subfiles = 32 + flags = FLAG_IS_MULTIFILE + + header = _default_header(flags=flags, n_points=n_points, n_subfiles=n_subfiles) + header["first_x"] = 200.0 + header["last_x"] = 800.0 + header["x_unit_code"] = 3 # Wavelength (nm) + header["y_unit_code"] = 2 # Absorbance + header["z_unit_code"] = 5 # Time (min) + + rng = np.random.default_rng(789) + + parts: list[bytes] = [_pack_header_le(header)] + for i in range(n_subfiles): + y = rng.normal(loc=0.0, scale=0.15, size=n_points).astype(np.float64) + subhdr = { + "flags": 0, + "exponent": 0, + "subfile_index": i, + "z_value": float(i), + "z_next": float(i + 1), + "noise": 0.0, + "n_points": 0, + "n_scans": 0, + "w_value": 0.0, + "reserved": b"\x00" * 4, + } + parts.append(_pack_subheader_le(subhdr)) + parts.append(_encode_y_fixed_int32(y, exponent=int(header["exponent"]))) + + return b"".join(parts) + + +def make_shared_x_4d_map() -> bytes: + # Tests expect w_planes=11 and len=121 + n_points = 64 + n_subfiles = 121 + w_planes = 11 + flags = FLAG_IS_MULTIFILE + + header = _default_header(flags=flags, n_points=n_points, n_subfiles=n_subfiles) + header["w_planes"] = w_planes + header["x_unit_code"] = 0 + header["y_unit_code"] = 0 + + rng = np.random.default_rng(2468) + + # 121 subfiles = 11 planes * 11 z positions + z_per_plane = n_subfiles // w_planes + + parts: list[bytes] = [_pack_header_le(header)] + for i in range(n_subfiles): + plane = i // z_per_plane + y = rng.normal(loc=0.0, scale=0.1, size=n_points).astype(np.float64) + subhdr = { + "flags": 0, + "exponent": 0, + "subfile_index": i, + "z_value": float(i % z_per_plane), + "z_next": float((i % z_per_plane) + 1), + "noise": 0.0, + "n_points": 0, + "n_scans": 0, + "w_value": float(plane), + "reserved": b"\x00" * 4, + } + parts.append(_pack_subheader_le(subhdr)) + parts.append(_encode_y_fixed_int32(y, exponent=int(header["exponent"]))) + + return b"".join(parts) + + +def make_log_text_block(text: str) -> bytes: + # Reader expects 64-byte LOGSTC header; it unpacks first 20 bytes as 5 uint32. + text_bytes = text.encode("latin-1", errors="replace") + txt_offset = 64 + logsize = 64 + len(text_bytes) + 1 # include NUL terminator + + logstc = bytearray(64) + struct.pack_into(" bytes: + # Modelled after ft-ir.spc tests. + n_points = 256 + header = _default_header(flags=0, n_points=n_points, n_subfiles=1) + header["first_x"] = 447.48 + header["last_x"] = 4002.28 + header["x_unit_code"] = 1 # Wavenumber + header["y_unit_code"] = 128 # Transmittance + header["date_int"] = _pack_date_int(1995, 4, 18, 9, 20) + + rng = np.random.default_rng(1357) + y = rng.normal(loc=0.0, scale=0.12, size=n_points).astype(np.float64) + + subhdr = { + "flags": 0, + "exponent": 0, + "subfile_index": 0, + "z_value": 0.0, + "z_next": 0.0, + "noise": 0.0, + "n_points": 0, + "n_scans": 0, + "w_value": 0.0, + "reserved": b"\x00" * 4, + } + + # Build payload without log first so we can set log_offset. + payload = b"".join([ + _pack_header_le(header), + _pack_subheader_le(subhdr), + _encode_y_fixed_int32(y, exponent=int(header["exponent"])), + ]) + + # Create deterministic log text with exact length expected by tests. + # Tests assert startswith("MODEL") and len == 376. + base = "MODEL=synthetic\r\nSCANS=1\r\n" + if len(base) > 376: + raise ValueError("Base log text unexpectedly long") + log_text = base + ("X" * (376 - len(base))) + + log_block = make_log_text_block(log_text) + + # Patch log_offset in header. + log_offset = len(payload) + header["log_offset"] = log_offset + + payload_with_log = b"".join([ + _pack_header_le(header), + _pack_subheader_le(subhdr), + _encode_y_fixed_int32(y, exponent=int(header["exponent"])), + log_block, + ]) + + return payload_with_log + + +def make_xyxy_single_ms() -> bytes: + # Single-subfile TXYXYS set: reader will still expose spc.x/spc.y. + n_points = 50 + flags = FLAG_Y_16BIT | FLAG_CUSTOM_AXIS_LABELS | FLAG_PER_SUBFILE_XY | FLAG_EXPLICIT_X + + header = _default_header(flags=flags, n_points=0, n_subfiles=1) + header["x_unit_code"] = 9 # Mass (m/z) + header["y_unit_code"] = 0 + header["axis_label_text"] = _cstr("X\x00Y\x00", 30) + + x = np.linspace(10.0, 100.0, n_points, dtype=np.float64).astype(" bytes: + flags = FLAG_IS_MULTIFILE | FLAG_PER_SUBFILE_XY | FLAG_EXPLICIT_X + + n_subfiles = 3 + header = _default_header(flags=flags, n_points=0, n_subfiles=n_subfiles) + header["x_unit_code"] = 0 + header["y_unit_code"] = 0 + + rng = np.random.default_rng(999) + + parts: list[bytes] = [_pack_header_le(header)] + for i, n_points in enumerate((10, 12, 8)): + x = np.linspace(1.0, float(n_points), n_points, dtype=np.float64).astype(" bytes: + # The reader rejects version 0x4D before parsing. We only need a 512-byte buffer + # with byte[1] = 0x4D. + buf = bytearray(b"\x00" * 512) + buf[1] = 0x4D + return bytes(buf) + + +@dataclass(frozen=True) +class SyntheticDataset: + """A synthetic dataset that mimics tests/data filenames.""" + + files: dict[str, bytes] + + +def build_synthetic_dataset() -> SyntheticDataset: + return SyntheticDataset( + files={ + "s_evenx.spc": make_shared_x_single_evenx(), + "s_xy.spc": make_shared_x_single_explicit_x(), + "m_evenz.spc": make_shared_x_multifile_evenz(), + "nir.spc": make_shared_x_multifile_evenz(), + "4d_map.spc": make_shared_x_4d_map(), + "ft-ir.spc": make_shared_x_single_with_log(), + "raman.spc": make_shared_x_single_with_log(), + "nmr_fid.spc": make_shared_x_single_with_log(), + "nmr_spc.spc": make_shared_x_single_with_log(), + "ms.spc": make_xyxy_single_ms(), + "m_xyxy.spc": make_xyxy_multifile_m_xyxy(), + "m_ordz.spc": make_old_format_stub(), + } + ) + + +def write_synthetic_dataset(dir_path: Path) -> Path: + dir_path.mkdir(parents=True, exist_ok=True) + dataset = build_synthetic_dataset() + for name, payload in dataset.files.items(): + (dir_path / name).write_bytes(payload) + + # Keep the directory looking like tests/data. + (dir_path / "testdata.md").write_text( + "Synthetic dataset generated at test time.\n", + encoding="utf-8", + ) + + return dir_path + + +def choose_data_dir(*, tmp_path_factory, real_data_dir: Path) -> Path: + """Return a directory containing .spc files for tests. + + Preference order: + 1) Use real_data_dir if it looks populated. + 2) Otherwise generate a synthetic dataset into a temp directory. + + You can force synthetic fixtures by setting SPC_USE_SYNTHETIC_DATA=1. + """ + + force = os.environ.get("SPC_USE_SYNTHETIC_DATA", "").strip() == "1" + + # "Looks populated" means at least a couple of key files exist. + has_real = (real_data_dir / "s_evenx.spc").is_file() and (real_data_dir / "m_evenz.spc").is_file() + + if has_real and not force: + return real_data_dir + + synth_dir = tmp_path_factory.mktemp("spc_synthetic_data") + return write_synthetic_dataset(Path(synth_dir)) diff --git a/tests/test_bad_inputs.py b/tests/test_bad_inputs.py index 9b73307..c9363d2 100644 --- a/tests/test_bad_inputs.py +++ b/tests/test_bad_inputs.py @@ -5,11 +5,13 @@ import pytest from spcfile import SPCFile +from tests.fixtures_synthetic import choose_data_dir @pytest.fixture(scope="session") -def data_dir() -> Path: - return Path(__file__).parent / "data" +def data_dir(tmp_path_factory) -> Path: + real = Path(__file__).parent / "data" + return choose_data_dir(tmp_path_factory=tmp_path_factory, real_data_dir=real) def test_truncated_header_raises_clear_value_error(tmp_path: Path, data_dir: Path) -> None: diff --git a/tests/test_spcfile.py b/tests/test_spcfile.py index 976e055..5562583 100644 --- a/tests/test_spcfile.py +++ b/tests/test_spcfile.py @@ -6,11 +6,13 @@ from spcfile import SPCFile, SPCSubfile from spcfile.spcfile import FLAG_EXPLICIT_X +from tests.fixtures_synthetic import choose_data_dir @pytest.fixture(scope="session") -def data_dir() -> Path: - return Path(__file__).parent / "data" +def data_dir(tmp_path_factory) -> Path: + real = Path(__file__).parent / "data" + return choose_data_dir(tmp_path_factory=tmp_path_factory, real_data_dir=real) class TestSPCFileConstruction: