From ae1ae3d69413f2a957db68ecfebfc8aaea00be2f Mon Sep 17 00:00:00 2001 From: Alexander Gwosdz Date: Tue, 1 Sep 2026 16:30:10 -0400 Subject: [PATCH 1/2] Improve JMP17 writer and reader fixture coverage --- examples/bogus_data.jmp | Bin 0 -> 1312 bytes examples/create_bogus_data.jsl | 43 +++++ examples/empty_data.jmp | Bin 0 -> 1080 bytes src/jmpio/column.py | 102 +++++++----- src/jmpio/metadata.py | 2 +- src/jmpio/writer.py | 252 +++++++++++++---------------- tests/test_data/bugMWE4.jmp | Bin 0 -> 12651 bytes tests/test_data/compact_UInt16.jmp | Bin 0 -> 132258 bytes tests/test_data/compact_UInt32.jmp | Bin 0 -> 232756 bytes tests/test_data/compact_UInt8.jmp | Bin 0 -> 2677 bytes tests/test_data/minusfour.jmp | Bin 0 -> 1802 bytes tests/test_reader.py | 35 +++- tests/test_writer.py | 57 +++++-- 13 files changed, 298 insertions(+), 193 deletions(-) create mode 100644 examples/bogus_data.jmp create mode 100644 examples/create_bogus_data.jsl create mode 100644 examples/empty_data.jmp create mode 100644 tests/test_data/bugMWE4.jmp create mode 100644 tests/test_data/compact_UInt16.jmp create mode 100644 tests/test_data/compact_UInt32.jmp create mode 100644 tests/test_data/compact_UInt8.jmp create mode 100644 tests/test_data/minusfour.jmp diff --git a/examples/bogus_data.jmp b/examples/bogus_data.jmp new file mode 100644 index 0000000000000000000000000000000000000000..c3172758280200ae815f2d0c295302f8fe5ae104 GIT binary patch literal 1312 zcmbVLO=uHA6n>ke*{1EO3gSVTi=~A$Y?79mf^C|Lp(VxGSgl2Er<+N4akCqDCn@y@ zPu>ItPkQw#f+vdyuYxGzspp;*?;do0Gdl}G>7_5cyl>~t`+pOL06GAqkTv`zU=&eA zX-FgJ^48>el=^k*ySBSSj-W&WU{U%Ql;-$}G;`Im1Jb0N5>B@rMl9MS+l;v+pgqWO=2YJ5zTU4H#*9(v1im-IF=>#X1@;G-yv$FWpC_s1*JShq*ZpX|jO( z-k!`+)Sr`m9#9b?nSxvR-QQ;&qVB=X=LbI)MxluM<*q{tMN%*d#VpB}W{uKpemYB5 znJg+RD$J5=s^a`iu8=bZ5T^m~6rO2c<*YQ+3Xnt=>o5mXFa}j9 z!JjZ*iX=Qi%_j1D}D#N73{lRPupviu6;-x6xJ1qe8i9iQV~^Qb6QU zq#Bl~mEn|>3Dk-6IhavqS_VNzP~~!UtubjB`4|*pP>jJ;45kNRg>vTG)H!YnQ;_dZ zLDw@Gz3xzxZF`HJiyPpqvKI51YqCX)Iu3Jp*hM=qsoxxcg&?4N8Vn_F$4ykeurcEj z_E*0JCsg%O7eIAk48D9us@*6{`Fg!9O}~ZV%NIzeH_B4}S}%{lQoGajZN9hJF*ycK zs=yjlRAe2*V;3a0rO{y~so8Ct2jll^joSFMF_9%}2=WDi8kV;k>y3$r6CqMsT1LAB z5Djh%Mr7)mDj}%`8JAFn{j-n1U)ldKKXmY_b7APClJDkaDp<7#AEsb{)T};^gaZ z9EBK+fjqC$90em|1tS9^V;u!UODh9QD??Kqg`m`&)WqUc9R)An03C&})S}|d{5%Ci zb3G$Hga5#2g#n0FFdl?v1X7@=W(HzV)UyIH$PpSqn84rxgb_fD4}eT9g{GDbQ2H&D zHUt?91O-rfBb5FErCAxA5=)XZIKj%_@B4R33ZkBY7fQ=_b1{%Q92TQ&Zv{L0%xk2u=o%{G62d Qkjz|=8**|;b^|ad0ld0zy8r+H literal 0 HcmV?d00001 diff --git a/src/jmpio/column.py b/src/jmpio/column.py index fc5c512..d9eef17 100644 --- a/src/jmpio/column.py +++ b/src/jmpio/column.py @@ -203,19 +203,19 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any: if len(row_data_bytes) < width: raise EOFError(f"Not enough data for row state in {column_name}") - marker_idx = bit_cat(row_data_bytes[6], row_data_bytes[7]) - marker = ROWSTATE_MARKERS[marker_idx] if marker_idx < len(ROWSTATE_MARKERS) else chr(marker_idx) - - r, g, b = 0.0, 0.0, 0.0 - if row_data_bytes[4] == 0xFF: - r = row_data_bytes[3] / 255.0 - g = row_data_bytes[2] / 255.0 - b = row_data_bytes[1] / 255.0 - else: - color_idx = row_data_bytes[1] - if 0 <= color_idx < len(ROWSTATE_COLORS): - hex_color = ROWSTATE_COLORS[color_idx] - r, g, b = hex_to_rgb(hex_color) + marker_idx = bit_cat(row_data_bytes[6], row_data_bytes[5]) + marker = ROWSTATE_MARKERS[marker_idx] if marker_idx < len(ROWSTATE_MARKERS) else chr(marker_idx) + + r, g, b = 0.0, 0.0, 0.0 + if row_data_bytes[3] == 0xFF: + r = row_data_bytes[2] / 255.0 + g = row_data_bytes[1] / 255.0 + b = row_data_bytes[0] / 255.0 + else: + color_idx = row_data_bytes[0] + if 0 <= color_idx < len(ROWSTATE_COLORS): + hex_color = ROWSTATE_COLORS[color_idx] + r, g, b = hex_to_rgb(hex_color) row_states.append(RowState(marker=marker, color=(r, g, b))) return pd.Series(row_states) @@ -239,12 +239,12 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any: if len(all_string_data) < data_size: raise EOFError(f"Not enough data for const char column {column_name}") - for i in range(info.nrows): - start = i * width - s_bytes = all_string_data[start : start + width] - s = s_bytes.rstrip(b"\x00").decode("utf-8", errors="replace") - strings.append(s) - return pd.Series(strings) + for i in range(info.nrows): + start = i * width + s_bytes = all_string_data[start : start + width] + s = s_bytes.split(b"\x00", 1)[0].decode("utf-8", errors="replace") + strings.append(s) + return pd.Series(strings) # Variable width elif [dt3, dt4, dt5] == [0x00, 0x00, 0x00]: @@ -265,14 +265,17 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any: # Indices to pool wb = buf.read(1)[0] - if wb == 1: - idx_dtype = np.int8 - idx_itemsize = 1 - elif wb == 2: - idx_dtype = np.int16 - idx_itemsize = 2 - else: - raise ValueError(f"Unknown index width byte {wb} for pooled var char in {column_name}") + if wb == 1: + idx_dtype = np.uint8 + idx_itemsize = 1 + elif wb == 2: + idx_dtype = np.dtype(" Any: return pd.Series(strings) # Non-pooled compressed variable-width strings - data_payload_source.seek(9) - width_bytes_val = data_payload_source.read(1)[0] + data_payload_source.seek(8) + width_bytes_val = data_payload_source.read(1)[0] lengths_offset_in_payload = 13 data_payload_source.seek(lengths_offset_in_payload) @@ -320,12 +323,13 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any: lengths = np.frombuffer(lengths_raw, dtype=len_dtype) string_data_block = data_payload_source.read() current_offset_in_strings = 0 - for length in lengths: - s_bytes = string_data_block[current_offset_in_strings : current_offset_in_strings + length] - strings.append(s_bytes.decode("utf-8", errors="replace")) - current_offset_in_strings += int(length) - return pd.Series(strings) - else: + for length in lengths: + length_int = int(length) + s_bytes = string_data_block[current_offset_in_strings : current_offset_in_strings + length_int] + strings.append(s_bytes.decode("utf-8", errors="replace")) + current_offset_in_strings += length_int + return pd.Series(strings) + else: # Uncompressed variable width file.seek(start_pos_after_dt) _ = file.read(6) @@ -355,11 +359,29 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any: file.seek(col_end - sum_lengths) all_string_data_bytes = file.read(sum_lengths) current_offset = 0 - for length in lengths: - s_bytes = all_string_data_bytes[current_offset : current_offset + length] - strings.append(s_bytes.decode("utf-8", errors="replace")) - current_offset += int(length) - return pd.Series(strings) + for length in lengths: + length_int = int(length) + s_bytes = all_string_data_bytes[current_offset : current_offset + length_int] + strings.append(s_bytes.decode("utf-8", errors="replace")) + current_offset += length_int + return pd.Series(strings) + + elif dt1 == 0x03 and dt2 == 0x03: + if not is_compressed: + file.seek(start_pos_after_dt) + uncompressed_data_block_bytes = file.read(col_end - start_pos_after_dt) + data_payload_source = io.BytesIO(uncompressed_data_block_bytes) + + if data_payload_source is None: + raise ValueError("data_payload_source not initialized for Int64 row state type") + + width = dt5 + data_size = width * info.nrows + data_payload_source.seek(-data_size, 2) + raw_data = data_payload_source.read(data_size) + if len(raw_data) < data_size: + raise EOFError(f"Not enough data for Int64 row state column {column_name}") + return pd.Series(np.frombuffer(raw_data, dtype=np.dtype(" tuple[list[str], list[int]]: raise EOFError("Unexpected end of file when searching for column information") # Check for special marker bytes - if twobytes in [b"\xfd\xff", b"\xfe\xff", b"\xff\xff"]: + if twobytes in [b"\xfc\xff", b"\xfd\xff", b"\xfe\xff", b"\xff\xff"]: n = struct.unpack(" None: +def write_jmp(df: pd.DataFrame, filename: str, compress: bool = True, version: str = "17.2.0") -> None: """ Write a pandas DataFrame to a JMP file @@ -33,7 +33,7 @@ def write_jmp(df: pd.DataFrame, filename: str, compress: bool = True, version: s Path to the output file compress : bool, default=True Whether to compress the data - version : str, default="16.0" + version : str, default="17.2.0" JMP version to use in the file header Returns: @@ -60,21 +60,29 @@ def write_jmp(df: pd.DataFrame, filename: str, compress: bool = True, version: s if directory and not os.path.exists(directory): os.makedirs(directory) - # Open file in binary write mode - with open(filename, "wb") as file: - # Write file header - write_file_header(file, df, version) - - # Write column metadata - column_offsets = write_column_metadata(file, df) - - # Write column data - for i, column_name in enumerate(df.columns): - column_data = df[column_name] - write_column_data(file, column_data, column_offsets[i], column_name, compress) - - # Any final corrections or clean-up - finalize_file(file) + # Open file in binary write mode + with open(filename, "wb") as file: + # Write file header + write_file_header(file, df, version) + + # Write column metadata + offset_table_pos = write_column_metadata(file, df) + + # Write column data + column_offsets = [] + for i, column_name in enumerate(df.columns): + column_offsets.append(file.tell()) + column_data = df[column_name] + write_column_data(file, column_data, column_offsets[i], column_name, compress) + + end_pos = file.tell() + file.seek(offset_table_pos) + for offset in column_offsets: + file.write(struct.pack(" None: @@ -90,34 +98,27 @@ def write_file_header(file: BinaryIO, df: pd.DataFrame, version: str) -> None: version : str JMP version to use in the header """ - # Write magic bytes (signature) - file.write(MAGIC_JMP) - - # Write padding up to the row offset - padding_size = 368 - len(MAGIC_JMP) - padding_data = bytearray([0] * padding_size) - - # Add some metadata in the padding (this is reverse-engineered) - # Here we could add metadata like creation software, etc. - file.write(padding_data) + # JMPReader.jl finds the table metadata by scanning for the byte sequence + # written inside foo2 below and then backing up to offset 368. Keep the + # preamble size aligned with JMP 17 files observed in the fixture set. + file.write(MAGIC_JMP) + file.write(bytearray([0] * (368 - len(MAGIC_JMP)))) # Write number of rows (Int64) and columns (Int32) file.write(struct.pack(" None: # Write more unknown values (1 UInt16) file.write(struct.pack(" list[int]: +def write_column_metadata(file: BinaryIO, df: pd.DataFrame) -> int: """ Write metadata about columns @@ -146,14 +147,15 @@ def write_column_metadata(file: BinaryIO, df: pd.DataFrame) -> list[int]: Returns: -------- - list[int] - List of file offsets for each column's data + int + File position where the Int64 column-offset table starts """ # Write column metadata section marker file.write(b"\xff\xff") - # Write some zeros (observed format) - file.write(struct.pack("