diff --git a/examples/bogus_data.jmp b/examples/bogus_data.jmp new file mode 100644 index 0000000..c317275 Binary files /dev/null and b/examples/bogus_data.jmp differ diff --git a/examples/create_bogus_data.jsl b/examples/create_bogus_data.jsl new file mode 100644 index 0000000..d159410 --- /dev/null +++ b/examples/create_bogus_data.jsl @@ -0,0 +1,43 @@ +Names Default To Here( 1 ); + +out = "C:/GitHub/jmpio-python/examples/bogus_data_jmp17_native.jmp"; + +dt = New Table( "Bogus Data JMP17 Native", + Add Rows( 5 ), + New Column( + "Batch", + Character, + Nominal, + Set Values( {"BOGUS-001", "BOGUS-002", "BOGUS-003", "BOGUS-004", "BOGUS-005"} ) + ), + New Column( + "Material", + Character, + Nominal, + Set Values( {"Placebo", "Control", "Reference", "Challenge", "Discard"} ) + ), + New Column( + "Assay", + Numeric, + Continuous, + Format( "Fixed Dec", 10, 2 ), + Set Values( [101.20, 98.70, ., 104.40, 99.90] ) + ), + New Column( + "Impurity_pct", + Numeric, + Continuous, + Format( "Fixed Dec", 10, 3 ), + Set Values( [0.120, 0.180, 0.090, ., 0.210] ) + ), + New Column( + "Result", + Character, + Nominal, + Set Values( {"Pass", "Pass", "Investigate", "Pass", "Fail"} ) + ) +); + +dt << Save( out ); +Close( dt, No Save ); +Quit(); diff --git a/examples/empty_data.jmp b/examples/empty_data.jmp new file mode 100644 index 0000000..55918bd Binary files /dev/null and b/examples/empty_data.jmp differ diff --git a/src/jmpio/column.py b/src/jmpio/column.py index fc5c512..d9eef17 100644 --- a/src/jmpio/column.py +++ b/src/jmpio/column.py @@ -203,19 +203,19 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any: if len(row_data_bytes) < width: raise EOFError(f"Not enough data for row state in {column_name}") - marker_idx = bit_cat(row_data_bytes[6], row_data_bytes[7]) - marker = ROWSTATE_MARKERS[marker_idx] if marker_idx < len(ROWSTATE_MARKERS) else chr(marker_idx) - - r, g, b = 0.0, 0.0, 0.0 - if row_data_bytes[4] == 0xFF: - r = row_data_bytes[3] / 255.0 - g = row_data_bytes[2] / 255.0 - b = row_data_bytes[1] / 255.0 - else: - color_idx = row_data_bytes[1] - if 0 <= color_idx < len(ROWSTATE_COLORS): - hex_color = ROWSTATE_COLORS[color_idx] - r, g, b = hex_to_rgb(hex_color) + marker_idx = bit_cat(row_data_bytes[6], row_data_bytes[5]) + marker = ROWSTATE_MARKERS[marker_idx] if marker_idx < len(ROWSTATE_MARKERS) else chr(marker_idx) + + r, g, b = 0.0, 0.0, 0.0 + if row_data_bytes[3] == 0xFF: + r = row_data_bytes[2] / 255.0 + g = row_data_bytes[1] / 255.0 + b = row_data_bytes[0] / 255.0 + else: + color_idx = row_data_bytes[0] + if 0 <= color_idx < len(ROWSTATE_COLORS): + hex_color = ROWSTATE_COLORS[color_idx] + r, g, b = hex_to_rgb(hex_color) row_states.append(RowState(marker=marker, color=(r, g, b))) return pd.Series(row_states) @@ -239,12 +239,12 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any: if len(all_string_data) < data_size: raise EOFError(f"Not enough data for const char column {column_name}") - for i in range(info.nrows): - start = i * width - s_bytes = all_string_data[start : start + width] - s = s_bytes.rstrip(b"\x00").decode("utf-8", errors="replace") - strings.append(s) - return pd.Series(strings) + for i in range(info.nrows): + start = i * width + s_bytes = all_string_data[start : start + width] + s = s_bytes.split(b"\x00", 1)[0].decode("utf-8", errors="replace") + strings.append(s) + return pd.Series(strings) # Variable width elif [dt3, dt4, dt5] == [0x00, 0x00, 0x00]: @@ -265,14 +265,17 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any: # Indices to pool wb = buf.read(1)[0] - if wb == 1: - idx_dtype = np.int8 - idx_itemsize = 1 - elif wb == 2: - idx_dtype = np.int16 - idx_itemsize = 2 - else: - raise ValueError(f"Unknown index width byte {wb} for pooled var char in {column_name}") + if wb == 1: + idx_dtype = np.uint8 + idx_itemsize = 1 + elif wb == 2: + idx_dtype = np.dtype(" Any: return pd.Series(strings) # Non-pooled compressed variable-width strings - data_payload_source.seek(9) - width_bytes_val = data_payload_source.read(1)[0] + data_payload_source.seek(8) + width_bytes_val = data_payload_source.read(1)[0] lengths_offset_in_payload = 13 data_payload_source.seek(lengths_offset_in_payload) @@ -320,12 +323,13 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any: lengths = np.frombuffer(lengths_raw, dtype=len_dtype) string_data_block = data_payload_source.read() current_offset_in_strings = 0 - for length in lengths: - s_bytes = string_data_block[current_offset_in_strings : current_offset_in_strings + length] - strings.append(s_bytes.decode("utf-8", errors="replace")) - current_offset_in_strings += int(length) - return pd.Series(strings) - else: + for length in lengths: + length_int = int(length) + s_bytes = string_data_block[current_offset_in_strings : current_offset_in_strings + length_int] + strings.append(s_bytes.decode("utf-8", errors="replace")) + current_offset_in_strings += length_int + return pd.Series(strings) + else: # Uncompressed variable width file.seek(start_pos_after_dt) _ = file.read(6) @@ -355,11 +359,29 @@ def read_column_data(file: BinaryIO, info: JMPInfo, column_idx: int) -> Any: file.seek(col_end - sum_lengths) all_string_data_bytes = file.read(sum_lengths) current_offset = 0 - for length in lengths: - s_bytes = all_string_data_bytes[current_offset : current_offset + length] - strings.append(s_bytes.decode("utf-8", errors="replace")) - current_offset += int(length) - return pd.Series(strings) + for length in lengths: + length_int = int(length) + s_bytes = all_string_data_bytes[current_offset : current_offset + length_int] + strings.append(s_bytes.decode("utf-8", errors="replace")) + current_offset += length_int + return pd.Series(strings) + + elif dt1 == 0x03 and dt2 == 0x03: + if not is_compressed: + file.seek(start_pos_after_dt) + uncompressed_data_block_bytes = file.read(col_end - start_pos_after_dt) + data_payload_source = io.BytesIO(uncompressed_data_block_bytes) + + if data_payload_source is None: + raise ValueError("data_payload_source not initialized for Int64 row state type") + + width = dt5 + data_size = width * info.nrows + data_payload_source.seek(-data_size, 2) + raw_data = data_payload_source.read(data_size) + if len(raw_data) < data_size: + raise EOFError(f"Not enough data for Int64 row state column {column_name}") + return pd.Series(np.frombuffer(raw_data, dtype=np.dtype(" tuple[list[str], list[int]]: raise EOFError("Unexpected end of file when searching for column information") # Check for special marker bytes - if twobytes in [b"\xfd\xff", b"\xfe\xff", b"\xff\xff"]: + if twobytes in [b"\xfc\xff", b"\xfd\xff", b"\xfe\xff", b"\xff\xff"]: n = struct.unpack(" None: +from .types import RowState + +SUPPORTED_WRITE_VERSION = "17.2.0" + + +def write_jmp( + df: pd.DataFrame, + filename: str, + compress: bool = True, + version: str = SUPPORTED_WRITE_VERSION, +) -> None: """ Write a pandas DataFrame to a JMP file @@ -33,8 +40,8 @@ def write_jmp(df: pd.DataFrame, filename: str, compress: bool = True, version: s Path to the output file compress : bool, default=True Whether to compress the data - version : str, default="16.0" - JMP version to use in the file header + version : str, default="17.2.0" + JMP file version to write. Currently only "17.2.0" is supported. Returns: -------- @@ -52,29 +59,43 @@ def write_jmp(df: pd.DataFrame, filename: str, compress: bool = True, version: s >>> 'strings': ['a', 'bb', 'ccc', 'dddd'] >>> }) >>> - >>> # Write to a JMP file - >>> jmpio.write_jmp(df, 'output.jmp') - """ - # Create directory if it doesn't exist - directory = os.path.dirname(os.path.abspath(filename)) + >>> # Write to a JMP file + >>> jmpio.write_jmp(df, 'output.jmp') + """ + if version != SUPPORTED_WRITE_VERSION: + raise ValueError( + f"Unsupported JMP write version {version!r}; " + f"only {SUPPORTED_WRITE_VERSION!r} is currently supported" + ) + + # Create directory if it doesn't exist + directory = os.path.dirname(os.path.abspath(filename)) if directory and not os.path.exists(directory): os.makedirs(directory) - # Open file in binary write mode - with open(filename, "wb") as file: - # Write file header - write_file_header(file, df, version) - - # Write column metadata - column_offsets = write_column_metadata(file, df) - - # Write column data - for i, column_name in enumerate(df.columns): - column_data = df[column_name] - write_column_data(file, column_data, column_offsets[i], column_name, compress) - - # Any final corrections or clean-up - finalize_file(file) + # Open file in binary write mode + with open(filename, "wb") as file: + # Write file header + write_file_header(file, df, version) + + # Write column metadata + offset_table_pos = write_column_metadata(file, df) + + # Write column data + column_offsets = [] + for i, column_name in enumerate(df.columns): + column_offsets.append(file.tell()) + column_data = df[column_name] + write_column_data(file, column_data, column_offsets[i], column_name, compress) + + end_pos = file.tell() + file.seek(offset_table_pos) + for offset in column_offsets: + file.write(struct.pack(" None: @@ -90,34 +111,27 @@ def write_file_header(file: BinaryIO, df: pd.DataFrame, version: str) -> None: version : str JMP version to use in the header """ - # Write magic bytes (signature) - file.write(MAGIC_JMP) - - # Write padding up to the row offset - padding_size = 368 - len(MAGIC_JMP) - padding_data = bytearray([0] * padding_size) - - # Add some metadata in the padding (this is reverse-engineered) - # Here we could add metadata like creation software, etc. - file.write(padding_data) + # JMPReader.jl finds the table metadata by scanning for the byte sequence + # written inside foo2 below and then backing up to offset 368. Keep the + # preamble size aligned with JMP 17 files observed in the fixture set. + file.write(MAGIC_JMP) + file.write(bytearray([0] * (368 - len(MAGIC_JMP)))) # Write number of rows (Int64) and columns (Int32) file.write(struct.pack(" None: # Write more unknown values (1 UInt16) file.write(struct.pack(" list[int]: +def write_column_metadata(file: BinaryIO, df: pd.DataFrame) -> int: """ Write metadata about columns @@ -146,14 +160,15 @@ def write_column_metadata(file: BinaryIO, df: pd.DataFrame) -> list[int]: Returns: -------- - list[int] - List of file offsets for each column's data + int + File position where the Int64 column-offset table starts """ # Write column metadata section marker file.write(b"\xff\xff") - # Write some zeros (observed format) - file.write(struct.pack("