diff --git a/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp b/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp index 6172666b6d53..67a0288a843f 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_metadata.cpp @@ -11,7 +11,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -19,11 +21,16 @@ #include #include +#include +#include #include +#include #include #include +#include #include +#include #include #include #include @@ -84,6 +91,87 @@ auto write_file_data(cudf::size_type num_cols, return source_sink; } +/** @brief Counts logical host-read bytes without including filesystem or page-cache effects. */ +class PageIndexCountingDatasource : public cudf::io::datasource { + public: + explicit PageIndexCountingDatasource(std::vector const& data) + : source_{cudf::io::datasource::create(cudf::host_span{ + reinterpret_cast(data.data()), data.size()})} + { + } + + std::unique_ptr host_read(std::size_t offset, std::size_t size) override + { + auto result = source_->host_read(offset, size); + bytes_read_ += result->size(); + return result; + } + + std::size_t host_read(std::size_t offset, std::size_t size, uint8_t* dst) override + { + auto const result = source_->host_read(offset, size, dst); + bytes_read_ += result; + return result; + } + + [[nodiscard]] std::size_t size() const override { return source_->size(); } + [[nodiscard]] std::size_t bytes_read() const { return bytes_read_.load(); } + void reset() { bytes_read_ = 0; } + + private: + std::unique_ptr source_; + std::atomic bytes_read_{0}; +}; + +/** @brief Reuses the metadata benchmark input, removing only optional index references. */ +std::vector make_optional_index_data(cudf::size_type num_cols, + cudf::size_type num_row_groups, + std::string const& layout) +{ + auto source_sink = write_file_data(num_cols, num_row_groups, io_type::HOST_BUFFER, true); + auto sources = cudf::io::make_datasources(source_sink.make_source_info()); + auto const footer_buffer = cudf::io::parquet::fetch_footer_to_host(*sources.front()); + cudf::io::parquet::FileMetaData metadata; + cudf::io::parquet::detail::CompactProtocolReader cp(footer_buffer->data(), footer_buffer->size()); + cp.read(&metadata); + CUDF_EXPECTS(layout == "none" or layout == "offset_only" or layout == "mixed" or layout == "both", + "Unexpected page index layout"); + for (auto& rg : metadata.row_groups) { + for (auto& col : rg.columns) { + if (layout == "none" or layout == "offset_only") { + col.column_index_offset = 0; + col.column_index_length = 0; + } + if (layout == "none") { + col.offset_index_offset = 0; + col.offset_index_length = 0; + } + } + } + if (layout == "mixed") { + metadata.row_groups.front().columns.front().column_index_offset = 0; + metadata.row_groups.front().columns.front().column_index_length = 0; + } + + auto const original = sources.front()->host_read(0, sources.front()->size()); + auto const begin = reinterpret_cast(original->data()); + std::vector data(begin, begin + original->size()); + cudf::io::parquet::file_ender_s ender; + CUDF_EXPECTS(data.size() >= sizeof(ender), "Invalid Parquet benchmark input"); + std::memcpy(&ender, data.data() + data.size() - sizeof(ender), sizeof(ender)); + CUDF_EXPECTS(ender.footer_len <= data.size() - sizeof(ender), "Invalid Parquet benchmark footer"); + data.resize(data.size() - sizeof(ender) - ender.footer_len); + // Keep unused index bytes in place so every layout has the same data-page offsets. + std::vector footer; + cudf::io::parquet::detail::CompactProtocolWriter writer(&footer); + writer.write(metadata); + data.insert(data.end(), footer.begin(), footer.end()); + ender.footer_len = static_cast(footer.size()); + auto const ender_bytes = reinterpret_cast(&ender); + data.insert(data.end(), ender_bytes, ender_bytes + sizeof(ender)); + return data; +} + // Combines `operands` into a balanced AST tree using `op`: pairing adjacent operands gives a tree // of depth ceil(log2(n)) rather than the n-deep chain a left fold would produce. [[nodiscard]] cudf::ast::expression const* reduce_balanced( @@ -343,6 +431,57 @@ void BM_parquet_filter_name_resolution(nvbench::state& state) mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage"); } +/** + * @brief Measures metadata parsing, index range calculation and index loading for both readers. + * + * Reports logical host-read bytes per invocation alongside metadata latency. Input generation is + * excluded from timing; host-buffer sources isolate metadata work from storage and cache behavior. + */ +void BM_parquet_page_index_metadata(nvbench::state& state) +{ + auto const num_cols = static_cast(state.get_int64("num_cols")); + auto const num_row_groups = static_cast(state.get_int64("num_row_groups")); + auto const hybrid = state.get_string("reader") == "hybrid"; + auto const data = make_optional_index_data(num_cols, num_row_groups, state.get_string("layout")); + auto source = std::make_unique(data); + auto const counter = source.get(); + std::vector> sources; + sources.emplace_back(std::move(source)); + auto const options = cudf::io::parquet_reader_options::builder().use_arrow_schema(false).build(); + state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().get())); + + state.exec( + nvbench::exec_tag::sync | nvbench::exec_tag::timer, [&](nvbench::launch&, auto& timer) { + counter->reset(); + timer.start(); + if (hybrid) { + auto const footer = cudf::io::parquet::fetch_footer_to_host(*counter); + auto const reader = cudf::io::parquet::experimental::hybrid_scan_reader{*footer, options}; + auto const range = reader.page_index_byte_range(); + if (not range.is_empty()) { + auto const indexes = cudf::io::parquet::fetch_page_index_to_host(*counter, range); + reader.setup_page_index(*indexes); + } + } else { + auto const metadata = cudf::io::read_parquet_footers(sources); + CUDF_EXPECTS(std::cmp_equal(metadata.front().row_groups.size(), num_row_groups), + "Unexpected number of row groups"); + } + timer.stop(); + }); + + state.add_buffer_size(counter->bytes_read(), "host_bytes_read", "Logical host bytes read"); + state.add_buffer_size(data.size(), "file_size", "Parquet file size"); +} + +NVBENCH_BENCH(BM_parquet_page_index_metadata) + .set_name("parquet_page_index_metadata") + .set_min_samples(4) + .add_string_axis("layout", {"none", "offset_only", "mixed", "both"}) + .add_string_axis("reader", {"regular", "hybrid"}) + .add_int64_axis("num_cols", {4, 16}) + .add_int64_axis("num_row_groups", {10, 100}); + NVBENCH_BENCH(BM_parquet_read_footer) .set_name("parquet_read_footer") .set_min_samples(4) diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp b/cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp index faba65f4f33a..ae5a4ec84bb4 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp @@ -31,6 +31,7 @@ using io::detail::inline_column_buffer; using parquet::detail::CompactProtocolReader; using parquet::detail::equality_literals_collector; using parquet::detail::input_column_info; +using parquet::detail::page_index_byte_range; using parquet::detail::row_group_info; using text::byte_range_info; @@ -76,45 +77,6 @@ namespace { return static_cast(total_row_groups); } -// Compute the page index (column index and/or offset index) byte range -[[nodiscard]] byte_range_info page_index_byte_range(FileMetaData const& file_metadata) -{ - auto const& row_groups = file_metadata.row_groups; - if (row_groups.empty() or row_groups.front().columns.empty()) { return {}; } - - // Helpers to check if a column chunk has a column index or offset index - auto const has_column_index = [](ColumnChunk const& col) { - return col.column_index_offset > 0 and col.column_index_length > 0; - }; - auto const has_offset_index = [](ColumnChunk const& col) { - return col.offset_index_offset > 0 and col.offset_index_length > 0; - }; - - auto const min_offset = [&]() -> int64_t { - auto const& first_col = row_groups.front().columns.front(); - if (has_column_index(first_col)) { - return first_col.column_index_offset; - } else if (has_offset_index(first_col)) { - return first_col.offset_index_offset; - } - return int64_t{0}; - }(); - - auto const max_offset = [&]() -> int64_t { - auto const& last_col = row_groups.back().columns.back(); - if (has_offset_index(last_col)) { - return last_col.offset_index_offset + last_col.offset_index_length; - } else if (has_column_index(last_col)) { - return last_col.column_index_offset + last_col.column_index_length; - } - return int64_t{0}; - }(); - - return (min_offset > 0 and max_offset > min_offset) - ? byte_range_info{min_offset, max_offset - min_offset} - : byte_range_info{}; -} - } // namespace metadata::metadata(cudf::host_span footer_bytes) diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index f229a6120721..40509a8d0b74 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -46,6 +47,34 @@ namespace cudf::io::parquet::detail { +// Compute the page index (column index and/or offset index) byte range +text::byte_range_info page_index_byte_range(FileMetaData const& file_metadata) +{ + int64_t min_offset = std::numeric_limits::max(); + int64_t max_offset = 0; + auto const include_index = [&](int64_t offset, int32_t length) { + if (offset > 0 and length > 0) { + CUDF_EXPECTS(offset <= std::numeric_limits::max() - length, + "Parquet page index range exceeds the supported offset range", + std::invalid_argument); + min_offset = std::min(min_offset, offset); + max_offset = std::max(max_offset, offset + length); + } + }; + + // Indexes are optional for each column chunk. The first and last chunks need not have either + // index, so inspect all chunks to include every index that setup_page_index will parse. + for (auto const& row_group : file_metadata.row_groups) { + for (auto const& column : row_group.columns) { + include_index(column.column_index_offset, column.column_index_length); + include_index(column.offset_index_offset, column.offset_index_length); + } + } + + return max_offset > min_offset ? text::byte_range_info{min_offset, max_offset - min_offset} + : text::byte_range_info{}; +} + std::size_t derive_pass_read_limit(std::size_t chunk_read_limit) { if (chunk_read_limit == 0) { return 0; } @@ -529,19 +558,22 @@ metadata::metadata(datasource* source, bool read_page_indexes) auto const has_strings = std::any_of( schema.begin(), schema.end(), [](auto const& elem) { return elem.type == Type::BYTE_ARRAY; }); - if (read_page_indexes and has_strings and not row_groups.empty() and - not row_groups.front().columns.empty()) { - // column index and offset index are encoded back to back. - // the first column of the first row group will have the first column index, the last - // column of the last row group will have the final offset index. - int64_t const min_offset = row_groups.front().columns.front().column_index_offset; - auto const& last_col = row_groups.back().columns.back(); - int64_t const max_offset = last_col.offset_index_offset + last_col.offset_index_length; - - if (max_offset > min_offset) { - size_t const length = max_offset - min_offset; - auto const page_idx_buf = source->host_read(min_offset, length); - setup_page_index({page_idx_buf->data(), length}, min_offset); + // Without offset indexes the decode paths cannot use column-index-derived information. + auto const has_offset_index = + std::any_of(row_groups.begin(), row_groups.end(), [](auto const& rg) { + return std::any_of(rg.columns.begin(), rg.columns.end(), [](auto const& col) { + return col.offset_index_offset > 0 and col.offset_index_length > 0; + }); + }); + + if (read_page_indexes and has_strings and has_offset_index) { + auto const page_index_range = page_index_byte_range(*this); + if (not page_index_range.is_empty()) { + auto const page_idx_buf = + source->host_read(page_index_range.offset(), page_index_range.size()); + CUDF_EXPECTS(std::cmp_equal(page_idx_buf->size(), page_index_range.size()), + "Encountered an invalid page index buffer"); + setup_page_index({page_idx_buf->data(), page_idx_buf->size()}, page_index_range.offset()); } } diff --git a/cpp/src/io/parquet/reader_impl_helpers.hpp b/cpp/src/io/parquet/reader_impl_helpers.hpp index 50279594164d..864753c06cd1 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.hpp +++ b/cpp/src/io/parquet/reader_impl_helpers.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -27,6 +28,16 @@ namespace cudf::io::parquet::detail { +/** + * @brief Computes the byte range containing the column and/or offset indexes. + * + * @throws std::invalid_argument if an index end exceeds the supported offset range + * + * @param file_metadata Parquet file metadata + * @return Page-index byte range, or an empty range when no indexes are available + */ +[[nodiscard]] text::byte_range_info page_index_byte_range(FileMetaData const& file_metadata); + /** * @brief page location and size info */ diff --git a/cpp/tests/io/experimental/hybrid_scan_test.cpp b/cpp/tests/io/experimental/hybrid_scan_test.cpp index 354e680fd5af..9fc667bdd5d4 100644 --- a/cpp/tests/io/experimental/hybrid_scan_test.cpp +++ b/cpp/tests/io/experimental/hybrid_scan_test.cpp @@ -32,6 +32,9 @@ #include +#include +#include + namespace { /** @@ -243,6 +246,126 @@ std::unique_ptr test_hybrid_scan_column_selection( // Base test fixture for tests struct HybridScanTest : public cudf::test::BaseFixture {}; +enum class PageIndexPresence { NONE, COLUMN_ONLY, OFFSET_ONLY, BOTH, MISSING_FIRST, MISSING_LAST }; + +struct HybridScanPageIndexTest : public HybridScanTest, + public ::testing::WithParamInterface {}; + +TEST_P(HybridScanPageIndexTest, OptionalIndexRanges) +{ + auto [written_table, parquet_buffer] = create_parquet_with_stats(); + auto const options = cudf::io::parquet_reader_options::builder().build(); + auto datasource = cudf::io::datasource::create(cudf::host_span{ + reinterpret_cast(parquet_buffer.data()), parquet_buffer.size()}); + auto const footer_buffer = cudf::io::parquet::fetch_footer_to_host(*datasource); + auto const seed_reader = + cudf::io::parquet::experimental::hybrid_scan_reader{*footer_buffer, options}; + auto file_metadata = seed_reader.parquet_metadata(); + ASSERT_EQ(file_metadata.row_groups.size(), 4); + ASSERT_EQ(file_metadata.row_groups.front().columns.size(), 3); + auto const presence = GetParam(); + auto const has_column_index = + presence != PageIndexPresence::NONE and presence != PageIndexPresence::OFFSET_ONLY; + auto const has_offset_index = + presence != PageIndexPresence::NONE and presence != PageIndexPresence::COLUMN_ONLY; + + for (auto& rg : file_metadata.row_groups) { + for (auto& col : rg.columns) { + ASSERT_GT(col.column_index_length, 0); + ASSERT_GT(col.offset_index_length, 0); + if (not has_column_index) { + col.column_index_offset = 0; + col.column_index_length = 0; + } + if (not has_offset_index) { + col.offset_index_offset = 0; + col.offset_index_length = 0; + } + } + } + auto& first = file_metadata.row_groups.front().columns.front(); + auto& last = file_metadata.row_groups.back().columns.back(); + if (presence == PageIndexPresence::MISSING_FIRST) { + first.column_index_offset = 0; + first.column_index_length = 0; + } + if (presence == PageIndexPresence::MISSING_LAST) { + last.offset_index_offset = 0; + last.offset_index_length = 0; + } + // The writer emits all column indexes before all offset indexes. Check both endpoints + // independently of the range helper, including missing indexes in the first and last chunks. + auto const index_start = + presence == PageIndexPresence::MISSING_FIRST + ? file_metadata.row_groups.front().columns[1].column_index_offset + : (has_column_index ? first.column_index_offset : first.offset_index_offset); + auto const& last_offset_chunk = + presence == PageIndexPresence::MISSING_LAST ? file_metadata.row_groups.back().columns[1] : last; + auto const index_end = + has_offset_index ? last_offset_chunk.offset_index_offset + last_offset_chunk.offset_index_length + : last.column_index_offset + last.column_index_length; + + // Supply the modified footer metadata through the public shared-metadata constructor. + auto const metadata = + cudf::io::parquet::experimental::hybrid_scan_metadata{std::move(file_metadata), options}; + auto const reader = cudf::io::parquet::experimental::hybrid_scan_reader{metadata}; + auto const range = reader.page_index_byte_range(); + ASSERT_EQ(range.offset(), index_start); + ASSERT_EQ(range.size(), index_end - index_start); + if (not range.is_empty()) { + auto const buffer = cudf::io::parquet::fetch_page_index_to_host(*datasource, range); + ASSERT_NO_THROW(reader.setup_page_index(*buffer)); + } + + auto const stream = cudf::get_default_stream(); + auto const mr = cudf::get_current_device_resource_ref(); + auto const row_groups = reader.all_row_groups(options); + auto const chunk_ranges = reader.all_column_chunks_byte_ranges(row_groups, options); + auto [buffers, data, tasks] = cudf::io::parquet::fetch_byte_ranges_to_device_async( + *datasource, chunk_ranges, cudf::io::parquet::io_submission_policy::SERIALIZE, stream, mr); + tasks.get(); + auto const result = reader.materialize_all_columns(row_groups, data, options, stream, mr); + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(written_table->view(), result.tbl->view()); +} + +INSTANTIATE_TEST_SUITE_P(IndexPresence, + HybridScanPageIndexTest, + ::testing::Values(PageIndexPresence::NONE, + PageIndexPresence::COLUMN_ONLY, + PageIndexPresence::OFFSET_ONLY, + PageIndexPresence::BOTH, + PageIndexPresence::MISSING_FIRST, + PageIndexPresence::MISSING_LAST)); + +struct HybridScanPageIndexOverflowTest : public HybridScanTest, + public ::testing::WithParamInterface {}; + +TEST_P(HybridScanPageIndexOverflowTest, RejectsOverflowingIndexRange) +{ + auto [written_table, parquet_buffer] = create_parquet_with_stats(); + auto const options = cudf::io::parquet_reader_options::builder().build(); + auto datasource = cudf::io::datasource::create(cudf::host_span{ + reinterpret_cast(parquet_buffer.data()), parquet_buffer.size()}); + auto const footer_buffer = cudf::io::parquet::fetch_footer_to_host(*datasource); + auto const seed_reader = + cudf::io::parquet::experimental::hybrid_scan_reader{*footer_buffer, options}; + auto file_metadata = seed_reader.parquet_metadata(); + auto& column = file_metadata.row_groups.front().columns.front(); + if (GetParam()) { + column.column_index_offset = std::numeric_limits::max() - 1; + column.column_index_length = 8; + } else { + column.offset_index_offset = std::numeric_limits::max() - 1; + column.offset_index_length = 8; + } + auto const metadata = + cudf::io::parquet::experimental::hybrid_scan_metadata{std::move(file_metadata), options}; + auto const reader = cudf::io::parquet::experimental::hybrid_scan_reader{metadata}; + EXPECT_THROW(std::ignore = reader.page_index_byte_range(), std::invalid_argument); +} + +INSTANTIATE_TEST_SUITE_P(IndexKind, HybridScanPageIndexOverflowTest, ::testing::Bool()); + TEST_F(HybridScanTest, FilterRowGroupsOnlyAndScanSelectColumns) { srand(0xc0ffee); diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index 8f61d2f2c07e..131fddeb20b4 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -29,11 +29,13 @@ #include +#include #include #include #include #include +#include #include #include #include @@ -78,6 +80,251 @@ TEST_F(ParquetReaderTest, ManyTinyStringPages) CUDF_TEST_EXPECT_TABLES_EQUAL(input, result.tbl->view()); } +namespace { + +class PageIndexTrackingDatasource : public cudf::io::datasource { + public: + PageIndexTrackingDatasource(std::vector const& data, + std::size_t index_offset, + std::size_t index_size) + : source_{cudf::io::datasource::create(cudf::host_span{ + reinterpret_cast(data.data()), data.size()})}, + index_offset_{index_offset}, + index_size_{index_size} + { + } + + std::unique_ptr host_read(std::size_t offset, std::size_t size) override + { + auto result = source_->host_read(offset, size); + record_read(offset, result->size()); + return result; + } + + std::size_t host_read(std::size_t offset, std::size_t size, uint8_t* dst) override + { + auto const bytes_read = source_->host_read(offset, size, dst); + record_read(offset, bytes_read); + return bytes_read; + } + + [[nodiscard]] std::size_t size() const override { return source_->size(); } + [[nodiscard]] std::size_t bytes_read() const { return bytes_read_.load(); } + [[nodiscard]] bool read_page_index() const { return read_page_index_.load(); } + + private: + void record_read(std::size_t offset, std::size_t size) + { + bytes_read_ += size; + if (index_size_ > 0 and offset == index_offset_ and size == index_size_) { + read_page_index_ = true; + } + } + + std::unique_ptr source_; + std::size_t const index_offset_; + std::size_t const index_size_; + std::atomic bytes_read_{0}; + std::atomic read_page_index_{false}; +}; + +} // namespace + +enum class PageIndexPresence { NONE, COLUMN_ONLY, OFFSET_ONLY, BOTH, MIXED }; + +struct ParquetPageIndexReadTest + : public ParquetReaderTest, + public ::testing::WithParamInterface> {}; + +TEST_P(ParquetPageIndexReadTest, ReadsOnlyUsableIndexes) +{ + // Pin the default footer hint so an environment override cannot turn this into a whole-file read. + tmp_env_var const footer_hint{"LIBCUDF_PARQUET_METADATA_SIZE_HINT", "65536"}; + auto const [presence, chunked] = GetParam(); + auto const has_column_index = presence == PageIndexPresence::COLUMN_ONLY or + presence == PageIndexPresence::BOTH or + presence == PageIndexPresence::MIXED; + auto const has_offset_index = presence == PageIndexPresence::OFFSET_ONLY or + presence == PageIndexPresence::BOTH or + presence == PageIndexPresence::MIXED; + auto constexpr rows_per_group = 2048; + auto constexpr num_groups = 4; + auto constexpr num_rows = rows_per_group * num_groups; + + // Keep the file larger than the speculative footer read, and disable dictionary encoding and + // compression so reading one row group requires substantially fewer bytes than the whole file. + std::vector strings; + strings.reserve(num_rows); + for (int i = 0; i < num_rows; ++i) { + strings.push_back(std::to_string(i) + std::string(128, 'x')); + } + cudf::test::strings_column_wrapper col(strings.begin(), strings.end()); + cudf::table_view const input{{col}}; + std::vector data; + auto const write_options = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&data}, input) + .row_group_size_rows(rows_per_group) + .max_page_fragment_size(rows_per_group) + .dictionary_policy(cudf::io::dictionary_policy::NEVER) + .compression(cudf::io::compression_type::NONE) + .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN) + .build(); + cudf::io::write_parquet(write_options); + + cudf::io::parquet::FileMetaData metadata; + read_footer(cudf::io::datasource::create(cudf::host_span{ + reinterpret_cast(data.data()), data.size()}), + &metadata); + ASSERT_EQ(metadata.row_groups.size(), num_groups); + + // Remove index references from the footer to cover different combinations of optional indexes. + // Leaving the unused index bytes in place preserves all data-page offsets. + for (auto& row_group : metadata.row_groups) { + for (auto& column : row_group.columns) { + ASSERT_GT(column.column_index_length, 0); + ASSERT_GT(column.offset_index_length, 0); + if (not has_column_index) { + column.column_index_offset = 0; + column.column_index_length = 0; + } + if (not has_offset_index) { + column.offset_index_offset = 0; + column.offset_index_length = 0; + } + } + } + + // A chunk without a column index can precede chunks that have one. Its offset index is after + // those column indexes, so using only the first chunk would omit indexes from the read buffer. + auto& first = metadata.row_groups.front().columns.front(); + if (presence == PageIndexPresence::MIXED) { + first.column_index_offset = 0; + first.column_index_length = 0; + } + auto const& first_column_index = + presence == PageIndexPresence::MIXED ? metadata.row_groups[1].columns.front() : first; + auto const& last = metadata.row_groups.back().columns.back(); + auto const index_start = + has_column_index ? first_column_index.column_index_offset : first.offset_index_offset; + auto const index_end = has_offset_index ? last.offset_index_offset + last.offset_index_length + : last.column_index_offset + last.column_index_length; + + cudf::io::parquet::file_ender_s ender; + std::memcpy(&ender, data.data() + data.size() - sizeof(ender), sizeof(ender)); + data.resize(data.size() - sizeof(ender) - ender.footer_len); + std::vector footer; + cudf::io::parquet::detail::CompactProtocolWriter writer(&footer); + writer.write(metadata); + data.insert(data.end(), footer.begin(), footer.end()); + ender.footer_len = static_cast(footer.size()); + auto const ender_bytes = reinterpret_cast(&ender); + data.insert(data.end(), ender_bytes, ender_bytes + sizeof(ender)); + + PageIndexTrackingDatasource source(data, index_start, index_end - index_start); + auto const read_options = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{&source}) + .row_groups({{1}}) + .build(); + auto result = [&]() { + if (chunked) { + cudf::io::chunked_parquet_reader reader(0, read_options); + auto chunk = reader.read_chunk(); + EXPECT_FALSE(reader.has_next()); + return chunk; + } + return cudf::io::read_parquet(read_options); + }(); + + auto const expected = cudf::slice(input, {rows_per_group, 2 * rows_per_group}).front(); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected, result.tbl->view()); + EXPECT_EQ(source.read_page_index(), has_offset_index); + EXPECT_LT(source.bytes_read(), data.size() / 2); +} + +INSTANTIATE_TEST_SUITE_P(IndexPresence, + ParquetPageIndexReadTest, + ::testing::Combine(::testing::Values(PageIndexPresence::NONE, + PageIndexPresence::COLUMN_ONLY, + PageIndexPresence::OFFSET_ONLY, + PageIndexPresence::BOTH, + PageIndexPresence::MIXED), + ::testing::Bool())); + +enum class InvalidPageIndex { PAST_EOF, COLUMN_OVERFLOW, OFFSET_OVERFLOW }; + +struct ParquetPageIndexInvalidRangeTest + : public ParquetReaderTest, + public ::testing::WithParamInterface> {}; + +TEST_P(ParquetPageIndexInvalidRangeTest, RejectsInvalidIndexRange) +{ + auto const [invalid_index, chunked] = GetParam(); + cudf::test::strings_column_wrapper col{"first", "second", "third"}; + cudf::table_view const input{{col}}; + std::vector data; + cudf::io::write_parquet( + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{&data}, input) + .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN) + .build()); + + cudf::io::parquet::FileMetaData metadata; + read_footer(cudf::io::datasource::create(cudf::host_span{ + reinterpret_cast(data.data()), data.size()}), + &metadata); + auto& column = metadata.row_groups.front().columns.front(); + ASSERT_GT(column.offset_index_offset, 0); + if (invalid_index == InvalidPageIndex::PAST_EOF) { + // Keep the index start within the file but make its advertised length exceed EOF. + column.offset_index_length = static_cast(data.size() * 2); + } else if (invalid_index == InvalidPageIndex::COLUMN_OVERFLOW) { + column.column_index_offset = std::numeric_limits::max() - 1; + column.column_index_length = 8; + } else { + column.offset_index_offset = std::numeric_limits::max() - 1; + column.offset_index_length = 8; + } + + cudf::io::parquet::file_ender_s ender; + std::memcpy(&ender, data.data() + data.size() - sizeof(ender), sizeof(ender)); + data.resize(data.size() - sizeof(ender) - ender.footer_len); + std::vector footer; + cudf::io::parquet::detail::CompactProtocolWriter writer(&footer); + writer.write(metadata); + data.insert(data.end(), footer.begin(), footer.end()); + ender.footer_len = static_cast(footer.size()); + auto const ender_bytes = reinterpret_cast(&ender); + data.insert(data.end(), ender_bytes, ender_bytes + sizeof(ender)); + if (invalid_index == InvalidPageIndex::PAST_EOF) { + ASSERT_GT(column.offset_index_offset + column.offset_index_length, data.size()); + } + + auto const options = + cudf::io::parquet_reader_options::builder( + cudf::io::source_info{cudf::host_span{data.data(), data.size()}}) + .build(); + // Reject malformed ranges before parsing indexes or attempting an overflowing read. + auto const read = [&] { + if (chunked) { + cudf::io::chunked_parquet_reader reader(0, options); + std::ignore = reader.read_chunk(); + } else { + std::ignore = cudf::io::read_parquet(options); + } + }; + if (invalid_index == InvalidPageIndex::PAST_EOF) { + EXPECT_THROW(read(), cudf::logic_error); + } else { + EXPECT_THROW(read(), std::invalid_argument); + } +} + +INSTANTIATE_TEST_SUITE_P(InvalidIndex, + ParquetPageIndexInvalidRangeTest, + ::testing::Combine(::testing::Values(InvalidPageIndex::PAST_EOF, + InvalidPageIndex::COLUMN_OVERFLOW, + InvalidPageIndex::OFFSET_OVERFLOW), + ::testing::Bool())); + TEST_F(ParquetReaderTest, UserBounds) { // trying to read more rows than there are should result in