Skip to content

Improve JMP17 writer and reader fixture coverage #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alexandergwosdz wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Improve JMP17 writer and reader fixture coverage #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ae1ae3d
Improve JMP17 writer and reader fixture coverage
Sep 1, 2026
0fba085
Reject unsupported JMP writer versions
Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added examples/bogus_data.jmp
View file
Open in desktop
Binary file not shown.
43 changes: 43 additions & 0 deletions examples/create_bogus_data.jsl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
Names Default To Here( 1 );

out = "C:/GitHub/jmpio-python/examples/bogus_data_jmp17_native.jmp";

dt = New Table( "Bogus Data JMP17 Native",
Add Rows( 5 ),
New Column(
"Batch",
Character,
Nominal,
Set Values( {"BOGUS-001", "BOGUS-002", "BOGUS-003", "BOGUS-004", "BOGUS-005"} )
),
New Column(
"Material",
Character,
Nominal,
Set Values( {"Placebo", "Control", "Reference", "Challenge", "Discard"} )
),
New Column(
"Assay",
Numeric,
Continuous,
Format( "Fixed Dec", 10, 2 ),
Set Values( [101.20, 98.70, ., 104.40, 99.90] )
),
New Column(
"Impurity_pct",
Numeric,
Continuous,
Format( "Fixed Dec", 10, 3 ),
Set Values( [0.120, 0.180, 0.090, ., 0.210] )
),
New Column(
"Result",
Character,
Nominal,
Set Values( {"Pass", "Pass", "Investigate", "Pass", "Fail"} )
)
);

dt << Save( out );
Close( dt, No Save );
Quit();
Binary file added examples/empty_data.jmp
View file
Open in desktop
Binary file not shown.
102 changes: 62 additions & 40 deletions src/jmpio/column.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,19 +203,19 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any:
if len(row_data_bytes) < width:
raise EOFError(f"Not enough data for row state in {column_name}")

marker_idx = bit_cat(row_data_bytes[6], row_data_bytes[7])
marker = ROWSTATE_MARKERS[marker_idx] if marker_idx < len(ROWSTATE_MARKERS) else chr(marker_idx)
r, g, b = 0.0, 0.0, 0.0
if row_data_bytes[4] == 0xFF:
r = row_data_bytes[3] / 255.0
g = row_data_bytes[2] / 255.0
b = row_data_bytes[1] / 255.0
else:
color_idx = row_data_bytes[1]
if 0 <= color_idx < len(ROWSTATE_COLORS):
hex_color = ROWSTATE_COLORS[color_idx]
r, g, b = hex_to_rgb(hex_color)
marker_idx = bit_cat(row_data_bytes[6], row_data_bytes[5])
marker = ROWSTATE_MARKERS[marker_idx] if marker_idx < len(ROWSTATE_MARKERS) else chr(marker_idx)

r, g, b = 0.0, 0.0, 0.0
if row_data_bytes[3] == 0xFF:
r = row_data_bytes[2] / 255.0
g = row_data_bytes[1] / 255.0
b = row_data_bytes[0] / 255.0
else:
color_idx = row_data_bytes[0]
if 0 <= color_idx < len(ROWSTATE_COLORS):
hex_color = ROWSTATE_COLORS[color_idx]
r, g, b = hex_to_rgb(hex_color)
row_states.append(RowState(marker=marker, color=(r, g, b)))
return pd.Series(row_states)

Expand All @@ -239,12 +239,12 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any:
if len(all_string_data) < data_size:
raise EOFError(f"Not enough data for const char column {column_name}")

for i in range(info.nrows):
start = i * width
s_bytes = all_string_data[start : start + width]
s = s_bytes.rstrip(b"\x00").decode("utf-8", errors="replace")
strings.append(s)
return pd.Series(strings)
for i in range(info.nrows):
start = i * width
s_bytes = all_string_data[start : start + width]
s = s_bytes.split(b"\x00", 1)[0].decode("utf-8", errors="replace")
strings.append(s)
return pd.Series(strings)

# Variable width
elif [dt3, dt4, dt5] == [0x00, 0x00, 0x00]:
Expand All @@ -265,14 +265,17 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any:

# Indices to pool
wb = buf.read(1)[0]
if wb == 1:
idx_dtype = np.int8
idx_itemsize = 1
elif wb == 2:
idx_dtype = np.int16
idx_itemsize = 2
else:
raise ValueError(f"Unknown index width byte {wb} for pooled var char in {column_name}")
if wb == 1:
idx_dtype = np.uint8
idx_itemsize = 1
elif wb == 2:
idx_dtype = np.dtype("<u2")
idx_itemsize = 2
elif wb == 4:
idx_dtype = np.dtype("<u4")
idx_itemsize = 4
else:
raise ValueError(f"Unknown index width byte {wb} for pooled var char in {column_name}")

idx_raw = buf.read(idx_itemsize * info.nrows)
idx = np.frombuffer(idx_raw, dtype=idx_dtype)
Expand Down Expand Up @@ -302,8 +305,8 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any:
return pd.Series(strings)

# Non-pooled compressed variable-width strings
data_payload_source.seek(9)
width_bytes_val = data_payload_source.read(1)[0]
data_payload_source.seek(8)
width_bytes_val = data_payload_source.read(1)[0]
lengths_offset_in_payload = 13
data_payload_source.seek(lengths_offset_in_payload)

Expand All @@ -320,12 +323,13 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any:
lengths = np.frombuffer(lengths_raw, dtype=len_dtype)
string_data_block = data_payload_source.read()
current_offset_in_strings = 0
for length in lengths:
s_bytes = string_data_block[current_offset_in_strings : current_offset_in_strings + length]
strings.append(s_bytes.decode("utf-8", errors="replace"))
current_offset_in_strings += int(length)
return pd.Series(strings)
else:
for length in lengths:
length_int = int(length)
s_bytes = string_data_block[current_offset_in_strings : current_offset_in_strings + length_int]
strings.append(s_bytes.decode("utf-8", errors="replace"))
current_offset_in_strings += length_int
return pd.Series(strings)
else:
# Uncompressed variable width
file.seek(start_pos_after_dt)
_ = file.read(6)
Expand Down Expand Up @@ -355,11 +359,29 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any:
file.seek(col_end - sum_lengths)
all_string_data_bytes = file.read(sum_lengths)
current_offset = 0
for length in lengths:
s_bytes = all_string_data_bytes[current_offset : current_offset + length]
strings.append(s_bytes.decode("utf-8", errors="replace"))
current_offset += int(length)
return pd.Series(strings)
for length in lengths:
length_int = int(length)
s_bytes = all_string_data_bytes[current_offset : current_offset + length_int]
strings.append(s_bytes.decode("utf-8", errors="replace"))
current_offset += length_int
return pd.Series(strings)

elif dt1 == 0x03 and dt2 == 0x03:
if not is_compressed:
file.seek(start_pos_after_dt)
uncompressed_data_block_bytes = file.read(col_end - start_pos_after_dt)
data_payload_source = io.BytesIO(uncompressed_data_block_bytes)

if data_payload_source is None:
raise ValueError("data_payload_source not initialized for Int64 row state type")

width = dt5
data_size = width * info.nrows
data_payload_source.seek(-data_size, 2)
raw_data = data_payload_source.read(data_size)
if len(raw_data) < data_size:
raise EOFError(f"Not enough data for Int64 row state column {column_name}")
return pd.Series(np.frombuffer(raw_data, dtype=np.dtype("<i8")))

except struct.error as e:
print(
Expand Down
2 changes: 1 addition & 1 deletion src/jmpio/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def column_info(file: BinaryIO, ncols: int) -> tuple[list[str], list[int]]:
raise EOFError("Unexpected end of file when searching for column information")

# Check for special marker bytes
if twobytes in [b"\xfd\xff", b"\xfe\xff", b"\xff\xff"]:
if twobytes in [b"\xfc\xff", b"\xfd\xff", b"\xfe\xff", b"\xff\xff"]:
n = struct.unpack("<q", file.read(8))[0] # Int64, little-endian
_ = file.read(n) # Skip this data
else:
Expand Down
Loading