diff --git a/benchmark/bench_accuracy_over_rowgroups.cpp b/benchmark/bench_accuracy_over_rowgroups.cpp index c99bdd89..d2d89091 100644 --- a/benchmark/bench_accuracy_over_rowgroups.cpp +++ b/benchmark/bench_accuracy_over_rowgroups.cpp @@ -88,9 +88,10 @@ void bench_accuracy_over_rowgroups() { } // Run the benchmark - auto size = benchmarker.bench(file_path, thread_specific_fls_dir_path); - const auto& footer_up = benchmarker.GetTableDescriptor(thread_specific_fls_dir_path); - const auto& first_rowgroup_descriptor = footer_up->m_rowgroup_descriptors[0]; + auto size = benchmarker.bench(file_path, thread_specific_fls_dir_path); + const auto& table_descriptor_handle = benchmarker.GetTableDescriptor(thread_specific_fls_dir_path); + const auto& first_rowgroup_descriptor = + table_descriptor_handle->Unpack()->m_rowgroup_descriptors[0]; { // Lock and store the main result diff --git a/benchmark/bench_compression_ratio.cpp b/benchmark/bench_compression_ratio.cpp index 08139db7..bfbc2c25 100644 --- a/benchmark/bench_compression_ratio.cpp +++ b/benchmark/bench_compression_ratio.cpp @@ -94,8 +94,9 @@ void run_compression_ratio_benchmark(const BenchmarkCase& benchmark_case) { main_results.emplace_back(table_name, fls_size, csv_size); } - const auto& tbl_desc = benchmarker.GetTableDescriptor(fls_dir); - const auto& rg_desc = tbl_desc->m_rowgroup_descriptors[0]; + const auto& table_descriptor_handle = benchmarker.GetTableDescriptor(fls_dir); + const auto& table_descriptor_object = table_descriptor_handle->Unpack(); + const auto& rg_desc = table_descriptor_object->m_rowgroup_descriptors[0]; { std::lock_guard lk(results_mutex); for (const auto& col : rg_desc->m_column_descriptors) { diff --git a/benchmark/bench_compression_time.cpp b/benchmark/bench_compression_time.cpp index 60ab3ace..86138a8e 100644 --- a/benchmark/bench_compression_time.cpp +++ b/benchmark/bench_compression_time.cpp @@ -21,7 +21,7 @@ class CompressionTimeBenchmarker : public CompressionRatioBenchmarker { auto start = std::chrono::high_resolution_clock::now(); for (n_t repetition_idx {0}; repetition_idx < n_repetitions; repetition_idx++) { - for (n_t vec_idx {0}; vec_idx < first_rowgroup_reader->get_descriptor().m_n_vec; vec_idx++) { + for (n_t vec_idx {0}; vec_idx < first_rowgroup_reader->get_descriptor().m_n_vec(); vec_idx++) { first_rowgroup_reader->get_chunk(vec_idx); }; } diff --git a/benchmark/bench_decompression_time.cpp b/benchmark/bench_decompression_time.cpp index c14fa440..75f4aa1e 100644 --- a/benchmark/bench_decompression_time.cpp +++ b/benchmark/bench_decompression_time.cpp @@ -21,7 +21,7 @@ class DecompressionTimeBenchmarker : public CompressionRatioBenchmarker { auto start = std::chrono::high_resolution_clock::now(); for (n_t repetition_idx {0}; repetition_idx < n_repetitions; repetition_idx++) { - for (n_t vec_idx {0}; vec_idx < first_rowgroup_reader->get_descriptor().m_n_vec; vec_idx++) { + for (n_t vec_idx {0}; vec_idx < first_rowgroup_reader->get_descriptor().m_n_vec(); vec_idx++) { first_rowgroup_reader->get_chunk(vec_idx); }; } diff --git a/benchmark/bench_random_access.cpp b/benchmark/bench_random_access.cpp index 8b0fc3b1..fff63169 100644 --- a/benchmark/bench_random_access.cpp +++ b/benchmark/bench_random_access.cpp @@ -18,9 +18,6 @@ class DecompressionTimeBenchmarker : public CompressionRatioBenchmarker { auto fls_reader = conn.reset().read_fls(dir_path / "data.fls"); auto first_rowgroup_reader = fls_reader->get_rowgroup_reader(0); - const auto rowgroup_up = std::make_unique(first_rowgroup_reader->get_descriptor(), conn); - // RandomAccessor random_accessor {*rowgroup_up}; - auto start = std::chrono::high_resolution_clock::now(); for (n_t repetition_idx {0}; repetition_idx < n_repetitions; repetition_idx++) { [[maybe_unused]] auto& expressions = first_rowgroup_reader->get_chunk(0); diff --git a/benchmark/bench_sample_size.cpp b/benchmark/bench_sample_size.cpp index c82e7996..94bc30ed 100644 --- a/benchmark/bench_sample_size.cpp +++ b/benchmark/bench_sample_size.cpp @@ -64,8 +64,9 @@ void bench_sampling() { auto size = benchmarker.bench_with_sample_size(file_path, thread_specific_fls_dir_path, sample_size); - const auto& table_descriptor = benchmarker.GetTableDescriptor(thread_specific_fls_dir_path); - const auto& first_rowgroup_descriptor = table_descriptor->m_rowgroup_descriptors[0]; + const auto& table_descriptor_handle = benchmarker.GetTableDescriptor(thread_specific_fls_dir_path); + const auto& table_descriptor_object = table_descriptor_handle->Unpack(); + const auto& first_rg_desc = table_descriptor_object->m_rowgroup_descriptors[0]; // Store the main result (thread-safe) { @@ -76,9 +77,9 @@ void bench_sampling() { // Store the detailed results (thread-safe) { std::lock_guard lock(results_mutex); - for (const auto& column_descriptor : first_rowgroup_descriptor->m_column_descriptors) { + for (const auto& column_descriptor : first_rg_desc->m_column_descriptors) { double bpt = static_cast(column_descriptor->total_size) / - (static_cast(first_rowgroup_descriptor->m_n_vec * CFG::VEC_SZ)); + (static_cast(first_rg_desc->m_n_vec * CFG::VEC_SZ)); double Bpt = bpt / 8; detailed_results.emplace_back(table_name, diff --git a/benchmark/include/benchmarker.hpp b/benchmark/include/benchmarker.hpp index 1790c3d2..68078f09 100644 --- a/benchmark/include/benchmarker.hpp +++ b/benchmark/include/benchmarker.hpp @@ -102,7 +102,7 @@ class CompressionRatioBenchmarker { } // Method to get the footer for the thread-specific directory - [[nodiscard]] up GetTableDescriptor(const path& thread_specific_fls_dir_path) const { + [[nodiscard]] up GetTableDescriptor(const path& thread_specific_fls_dir_path) const { return make_table_descriptor(thread_specific_fls_dir_path / TABLE_DESCRIPTOR_FILE_NAME); } @@ -133,12 +133,12 @@ class CompressionRatioBenchmarker { Write(table_path, thread_specific_fls_dir_path); vector result; auto table_descriptor = GetTableDescriptor(thread_specific_fls_dir_path); - auto& rowgroup_descriptor = table_descriptor->m_rowgroup_descriptors[0]; + auto& rowgroup_descriptor = *(*table_descriptor->Get()->m_rowgroup_descriptors())[0]; // Store the detailed results (thread-safe) { - for (const auto& column_descriptor : rowgroup_descriptor->m_column_descriptors) { - result.emplace_back(column_descriptor->encoding_rpn->operator_tokens[0]); + for (const auto& column_descriptor : *rowgroup_descriptor.m_column_descriptors()) { + result.emplace_back((*column_descriptor->encoding_rpn()->operator_tokens())[0]); } } diff --git a/benchmark/micro_benchmark_decompression.cpp b/benchmark/micro_benchmark_decompression.cpp index dcda2bd2..b46ec3f4 100644 --- a/benchmark/micro_benchmark_decompression.cpp +++ b/benchmark/micro_benchmark_decompression.cpp @@ -22,7 +22,7 @@ class DecompressionTimeBenchmarker : public CompressionRatioBenchmarker { auto start = benchmark::cycleclock::Now(); for (n_t repetition_idx {0}; repetition_idx < n_repetitions; repetition_idx++) { - for (n_t vec_idx {0}; vec_idx < rowgroup_reader->get_descriptor().m_n_vec; vec_idx++) { + for (n_t vec_idx {0}; vec_idx < rowgroup_reader->get_descriptor().m_n_vec(); vec_idx++) { rowgroup_reader->get_chunk(vec_idx); }; } @@ -30,7 +30,7 @@ class DecompressionTimeBenchmarker : public CompressionRatioBenchmarker { auto elapsed_cycles = end - start; return static_cast(elapsed_cycles) / - (static_cast(rowgroup_reader->get_descriptor().m_n_vec * CFG::VEC_SZ * n_repetitions)); + (static_cast(rowgroup_reader->get_descriptor().m_n_vec() * CFG::VEC_SZ * n_repetitions)); } public: diff --git a/mk/format.mk b/mk/format.mk index b75446a8..136e9857 100644 --- a/mk/format.mk +++ b/mk/format.mk @@ -23,9 +23,14 @@ FMT_CMD := python3 scripts/run-clang-format.py \ $(EXCLUDES) -.PHONY: format clang-format format-check rust-format-check +.PHONY: format clang-format format-check rust-format-check check-docker -clang-format: +# check if docker daemon is running +check-docker: + @docker info > /dev/null 2>&1 || \ + { $(call echo_error,❌ Docker is not running. Please start Docker and retry.); exit 1; } + +clang-format: check-docker $(call echo_start,Running clang-format with Docker…) $(DOCKER) $(UBUNTU_IMAGE) \ bash -c "apt update && apt install -y python3 clang-format-14 && \ @@ -33,11 +38,10 @@ clang-format: $(FMT_CMD) -i" $(call echo_done,clang-format run complete.) -format-check: +format-check: check-docker $(call echo_start,Checking formatting…) $(DOCKER) $(UBUNTU_IMAGE) \ bash -c "apt update && apt install -y clang-format-14 python3 && \ ln -s /usr/bin/clang-format-14 /usr/bin/clang-format && \ $(FMT_CMD)" $(call echo_done,Formatting check complete.) - diff --git a/mk/preamble.mk b/mk/preamble.mk index 1d11f8ba..f4962051 100644 --- a/mk/preamble.mk +++ b/mk/preamble.mk @@ -13,19 +13,25 @@ COLOR ?= 1 ifeq ($(COLOR),1) _Y := $(shell tput setaf 3 2>/dev/null || printf '\033[0;33m') _G := $(shell tput setaf 2 2>/dev/null || printf '\033[0;32m') + _E := $(shell tput setaf 1 2>/dev/null || printf '\033[0;31m') _R := $(shell tput sgr0 2>/dev/null || printf '\033[0m') else _Y := _G := + _E := _R := endif define echo_done - @printf '%s%s%s\n' '$(_G)' '$(1)' '$(_R)' + printf '%s%s%s\n' '$(_G)' '$(1)' '$(_R)' endef define echo_start - @printf '%s%s%s\n' '$(_Y)' '$(1)' '$(_R)' + printf '%s%s%s\n' '$(_Y)' '$(1)' '$(_R)' +endef + +define echo_error + printf '%s%s%s\n' '$(_E)' '$(1)' '$(_R)' endef # ── Root paths ────────────────────────────────────────── diff --git a/src/expression/cross_rle_operator.cpp b/src/expression/cross_rle_operator.cpp index 1f76bc54..638f3adb 100644 --- a/src/expression/cross_rle_operator.cpp +++ b/src/expression/cross_rle_operator.cpp @@ -147,14 +147,16 @@ void enc_cross_rle_opr::MoveSegments(vector>& segments * dec_cross_rle_opr \*--------------------------------------------------------------------------------------------------------------------*/ template -dec_cross_rle_opr::dec_cross_rle_opr(PhysicalExpr& physical_expr, +dec_cross_rle_opr::dec_cross_rle_opr(PhysicalExpr& /*physical_expr*/, const ColumnView& column_view, InterpreterState& state) - : values_segment( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 1])) - , lengths_segment( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 0])) { - state.cur_operand = state.cur_operand - 2; + : values_segment(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 1)]))) + , lengths_segment(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 0)]))) { + state.cur_operand -= 2; values_segment.PointTo(0); lengths_segment.PointTo(0); @@ -227,16 +229,19 @@ template struct dec_cross_rle_opr; template struct dec_cross_rle_opr; template struct dec_cross_rle_opr; -dec_cross_rle_opr::dec_cross_rle_opr(PhysicalExpr& physical_expr, +dec_cross_rle_opr::dec_cross_rle_opr(PhysicalExpr& /*physical_expr*/, const ColumnView& column_view, InterpreterState& state) - : values_bytes_seg( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 2])) - , values_offset_seg( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 1])) - , lengths_segment( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 0])) { - state.cur_operand = state.cur_operand - 3; + : values_bytes_seg(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 2)]))) + , values_offset_seg(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 1)]))) + , lengths_segment(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 0)]))) { + state.cur_operand -= 3; values_bytes_seg.PointTo(0); values_offset_seg.PointTo(0); diff --git a/src/expression/data_parallelize_patch_operator.cpp b/src/expression/data_parallelize_patch_operator.cpp index 70fed7a0..b53e8b72 100644 --- a/src/expression/data_parallelize_patch_operator.cpp +++ b/src/expression/data_parallelize_patch_operator.cpp @@ -11,7 +11,6 @@ #include "fls/expression/decoding_operator.hpp" #include "fls/expression/interpreter.hpp" #include "fls/expression/physical_expression.hpp" -#include "fls/reader/column_view.hpp" #include "fls/std/variant.hpp" #include "fls/table/rowgroup.hpp" #include diff --git a/src/expression/decoding_operator.cpp b/src/expression/decoding_operator.cpp index 4937ff55..ceb4e579 100644 --- a/src/expression/decoding_operator.cpp +++ b/src/expression/decoding_operator.cpp @@ -22,12 +22,14 @@ namespace fastlanes { template dec_unffor_opr::dec_unffor_opr(const ColumnView& column_view, InterpreterState& state) : bitpacked_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 2])) - , bw_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 1])) + column_view.GetSegment((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 2)])) + , bw_segment_view(column_view.GetSegment((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 1)])) , base_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 0])) { - state.cur_operand = state.cur_operand - 3; + column_view.GetSegment((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 0)])) { + state.cur_operand -= 3; } template @@ -93,15 +95,15 @@ template struct dec_uncompressed_opr; \*--------------------------------------------------------------------------------------------------------------------*/ template dec_constant_opr::dec_constant_opr(const ColumnView& column_view) { - FLS_ASSERT_E(column_view.column_descriptor.max->binary_data.size(), sizeof(PT)); - value = *reinterpret_cast(column_view.column_descriptor.max->binary_data.data()); + FLS_ASSERT_E(column_view.column_descriptor.max()->binary_data()->size(), sizeof(PT)); + value = *reinterpret_cast(column_view.column_descriptor.max()->binary_data()->data()); } dec_constant_str_opr::dec_constant_str_opr(const ColumnView& column_view) { - bytes.resize(column_view.column_descriptor.max->binary_data.size()); + bytes.resize(column_view.column_descriptor.max()->binary_data()->size()); memcpy(bytes.data(), - column_view.column_descriptor.max->binary_data.data(), - column_view.column_descriptor.max->binary_data.size()); + column_view.column_descriptor.max()->binary_data()->data(), + column_view.column_descriptor.max()->binary_data()->size()); }; template struct dec_constant_opr; @@ -118,10 +120,10 @@ template struct dec_constant_opr; /*--------------------------------------------------------------------------------------------------------------------*\ * dec_fls_str_uncompressed_opr \*--------------------------------------------------------------------------------------------------------------------*/ -dec_fls_str_uncompressed_opr::dec_fls_str_uncompressed_opr(const ColumnView& column_view, const RPNT& rpn) - : byte_arr_segment(column_view.GetSegment(rpn.operand_tokens[0])) - , length_segment(column_view.GetSegment(rpn.operand_tokens[1])) { - FLS_ASSERT_EQUALITY(rpn.operand_tokens.size(), 2); +dec_fls_str_uncompressed_opr::dec_fls_str_uncompressed_opr(const ColumnView& column_view, const RPN& rpn) + : byte_arr_segment(column_view.GetSegment((*rpn.operand_tokens())[0])) + , length_segment(column_view.GetSegment((*rpn.operand_tokens())[1])) { + FLS_ASSERT_EQUALITY(rpn.operand_tokens()->size(), 2); } void dec_fls_str_uncompressed_opr::PointTo(n_t vec_idx) { @@ -140,18 +142,18 @@ len_t* dec_fls_str_uncompressed_opr::Length() const { /*--------------------------------------------------------------------------------------------------------------------*\ * dec_struct_opr \*--------------------------------------------------------------------------------------------------------------------*/ -dec_struct_opr::dec_struct_opr(const ColumnDescriptorT& column_descriptor, - const ColumnView& column_view, +dec_struct_opr::dec_struct_opr(const ColumnDescriptor& column_descriptor, + const ColumnView& column_view, InterpreterState&, RowgroupReader& reader) { - auto& children = column_descriptor.children; + auto& children = *column_descriptor.children(); for (n_t children_idx {0}; children_idx < children.size(); ++children_idx) { - auto& child_column_descriptor = children[children_idx]; + auto& child_column_descriptor = *children[static_cast(children_idx)]; InterpreterState state; auto child_physical_expr = - make_decoding_expression(*child_column_descriptor, column_view.children[children_idx], reader, state); + make_decoding_expression(child_column_descriptor, column_view.children[children_idx], reader, state); internal_exprs.push_back(child_physical_expr); } diff --git a/src/expression/dict_expression.cpp b/src/expression/dict_expression.cpp index afa9996a..da30d1c0 100644 --- a/src/expression/dict_expression.cpp +++ b/src/expression/dict_expression.cpp @@ -141,11 +141,11 @@ template dec_dict_opr::dec_dict_opr(const PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) - : key_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 0])) + : key_segment_view(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 0)]))) , index_arr(nullptr) { - - state.cur_operand = state.cur_operand - 1; + state.cur_operand -= 1; visit(DictExprVisitor {index_arr}, physical_expr.operators[0]); } diff --git a/src/expression/frequency_operator.cpp b/src/expression/frequency_operator.cpp index 1eecc2f3..8fe99218 100644 --- a/src/expression/frequency_operator.cpp +++ b/src/expression/frequency_operator.cpp @@ -170,21 +170,26 @@ void enc_frequency_str_opr::MoveSegments(vector>& segments) { * dec_frequency_opr \*--------------------------------------------------------------------------------------------------------------------*/ template -dec_frequency_opr::dec_frequency_opr(PhysicalExpr& physical_expr, +dec_frequency_opr::dec_frequency_opr(PhysicalExpr& /*physical_expr*/, const ColumnView& column_view, InterpreterState& state) : frequent_value_seg( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 3])) + column_view.GetSegment((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 3)])) , exceptions_segment( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 2])) + column_view.GetSegment((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 2)])) , exceptions_position_segment( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 1])) + column_view.GetSegment((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 1)])) , n_exceptions_segment( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 0])) { - state.cur_operand = state.cur_operand - 4; + column_view.GetSegment((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 0)])) { + state.cur_operand -= 4; frequent_value_seg.PointTo(0); - frequent_val = *reinterpret_cast(frequent_value_seg.data); + const auto* fv_ptr = reinterpret_cast(frequent_value_seg.data); + frequent_val = *fv_ptr; } template @@ -232,27 +237,36 @@ template struct dec_frequency_opr; template struct dec_frequency_opr; template struct dec_frequency_opr; -dec_frequency_str_opr::dec_frequency_str_opr(PhysicalExpr& physical_expr, +dec_frequency_str_opr::dec_frequency_str_opr(PhysicalExpr& /*physical_expr*/, const ColumnView& column_view, InterpreterState& state) - : frequent_value_bytes_seg( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 5])) - , frequent_value_size_seg( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 4])) - , n_exceptions_seg( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 3])) - , exception_positions_seg( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 2])) - , exception_values_bytes_seg( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 1])) - , exception_values_offset_seg( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 0])) { - state.cur_operand = state.cur_operand - 6; + : frequent_value_bytes_seg(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 5)]))) + , frequent_value_size_seg(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 4)]))) + , n_exceptions_seg(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 3)]))) + , exception_positions_seg(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 2)]))) + , exception_values_bytes_seg(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 1)]))) + , exception_values_offset_seg(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 0)]))) { + state.cur_operand -= 6; frequent_value_bytes_seg.PointTo(0); frequent_value_size_seg.PointTo(0); - frequent_val = fls_string_t(reinterpret_cast(frequent_value_bytes_seg.data), - *reinterpret_cast(frequent_value_size_seg.data)); + + auto* bytes = reinterpret_cast(frequent_value_bytes_seg.data); + const auto len = *reinterpret_cast(frequent_value_size_seg.data); + + frequent_val = fls_string_t(bytes, len); } void dec_frequency_str_opr::PointTo(n_t vec_idx) { diff --git a/src/expression/fsst12_dict_operator.cpp b/src/expression/fsst12_dict_operator.cpp index 314712f1..615bb2d7 100644 --- a/src/expression/fsst12_dict_operator.cpp +++ b/src/expression/fsst12_dict_operator.cpp @@ -126,13 +126,18 @@ template dec_fsst12_dict_opr::dec_fsst12_dict_opr(const PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) - : fsst12_header_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 2])) - , fsst12_bytes_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 1])) - , fsst12_offset_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 0])) + : fsst12_header_segment_view(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 2)]))) + , fsst12_bytes_segment_view(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 1)]))) + , fsst12_offset_segment_view(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 0)]))) , index_arr(nullptr) { + // consume three operands + state.cur_operand -= 3; visit(FSST12DictExprVisitor {index_arr}, physical_expr.operators[0]); tmp_string.resize(CFG::String::max_bytes_per_string); @@ -144,7 +149,7 @@ dec_fsst12_dict_opr::dec_fsst12_dict_opr(const PhysicalExpr& physical_ [[maybe_unused]] auto symbol_table_size = fsst12_import(&fsst12_decoder, reinterpret_cast(fsst12_header_segment_view.data)); - FLS_ASSERT_E(symbol_table_size, fsst12_header_segment_view.data_span.size()) + FLS_ASSERT_E(symbol_table_size, fsst12_header_segment_view.data_span.size()); } template diff --git a/src/expression/fsst_dict_operator.cpp b/src/expression/fsst_dict_operator.cpp index 8735e51d..595392cd 100644 --- a/src/expression/fsst_dict_operator.cpp +++ b/src/expression/fsst_dict_operator.cpp @@ -126,14 +126,16 @@ template dec_fsst_dict_opr::dec_fsst_dict_opr(const PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) - : fsst_header_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 2])) - , fsst_bytes_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 1])) - , fsst_offset_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 0])) + : fsst_header_segment_view(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 2)]))) + , fsst_bytes_segment_view(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 1)]))) + , fsst_offset_segment_view(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 0)]))) , index_arr(nullptr) { - visit(FSSTDictExprVisitor {index_arr}, physical_expr.operators[0]); tmp_string.resize(CFG::String::max_bytes_per_string); @@ -144,7 +146,9 @@ dec_fsst_dict_opr::dec_fsst_dict_opr(const PhysicalExpr& physical_expr [[maybe_unused]] auto symbol_table_size = fsst_import(&fsst_decoder, reinterpret_cast(fsst_header_segment_view.data)); - FLS_ASSERT_E(symbol_table_size, fsst_header_segment_view.data_span.size()) + FLS_ASSERT_E(symbol_table_size, fsst_header_segment_view.data_span.size()); + + state.cur_operand -= 3; } template diff --git a/src/expression/interpreter.cpp b/src/expression/interpreter.cpp index 89c953c7..0e2f20a8 100644 --- a/src/expression/interpreter.cpp +++ b/src/expression/interpreter.cpp @@ -4,6 +4,7 @@ // src/expression/interpreter.cpp // ──────────────────────────────────────────────────────── #include "fls/expression/interpreter.hpp" +#include "flatbuffers/vector.h" // flatbuffers::Vector #include "fls/common/alias.hpp" #include "fls/common/assert.hpp" #include "fls/common/common.hpp" @@ -34,6 +35,8 @@ #include "fls/reader/column_view.hpp" #include "fls/std/type_traits.hpp" #include "fls/table/rowgroup.hpp" +#include // size_t +#include // uint32_t, uint64_t namespace fastlanes { /*--------------------------------------------------------------------------------------------------------------------*\ @@ -1055,28 +1058,38 @@ sp Interpreter::Encoding::Interpret(ColumnDescriptorT& column_desc * make_dec_uncompressed_expr \*--------------------------------------------------------------------------------------------------------------------*/ template -void make_dec_uncompressed_expr(PhysicalExpr& physical_expr, - const ColumnView& column_view, - const InterpreterState& state) { - auto& rpn = column_view.column_descriptor.encoding_rpn; +void make_dec_uncompressed_expr(PhysicalExpr& physical_expr, + const ColumnView& column_view, + const InterpreterState& /*state*/) { + const auto* rpn = column_view.column_descriptor.encoding_rpn(); + FLS_ASSERT_NOT_NULL_POINTER(rpn); - FLS_ASSERT_E(rpn->operand_tokens.size(), 1) + const auto* operands = rpn->operand_tokens(); + FLS_ASSERT_NOT_NULL_POINTER(operands); - physical_expr.operators.emplace_back( - make_shared>(column_view, rpn->operand_tokens.back())); + FLS_ASSERT_E(operands->size(), 1); + + const uint64_t last = operands->Get(operands->size() - 1); + + physical_expr.operators.emplace_back(std::make_shared>(column_view, last)); } /*--------------------------------------------------------------------------------------------------------------------*\ * make_dec_validitymask_expr \*--------------------------------------------------------------------------------------------------------------------*/ -void make_dec_validitymask_expr(PhysicalExpr& physical_expr, - const ColumnView& column_view, - const InterpreterState& state) { - auto& rpn = column_view.column_descriptor.encoding_rpn; +void make_dec_validitymask_expr(PhysicalExpr& physical_expr, + const ColumnView& column_view, + const InterpreterState& /*state*/) { + const auto* rpn = column_view.column_descriptor.encoding_rpn(); + FLS_ASSERT_NOT_NULL_POINTER(rpn); - FLS_ASSERT_E(rpn->operand_tokens.size(), 1) + const auto* operands = rpn->operand_tokens(); + FLS_ASSERT_NOT_NULL_POINTER(operands); - physical_expr.operators.emplace_back(make_shared(column_view, rpn->operand_tokens.back())); + FLS_ASSERT_E(operands->size(), 1); + + const uint64_t last = operands->Get(operands->size() - 1); + physical_expr.operators.emplace_back(std::make_shared(column_view, last)); } /*--------------------------------------------------------------------------------------------------------------------*\ @@ -1085,11 +1098,12 @@ void make_dec_validitymask_expr(PhysicalExpr& physical_expr, void make_dec_fls_str_uncompressed_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { + const auto* rpn = column_view.column_descriptor.encoding_rpn(); + FLS_ASSERT_NOT_NULL_POINTER(rpn); - auto& rpn = column_view.column_descriptor.encoding_rpn; + physical_expr.operators.emplace_back(std::make_shared(column_view, *rpn)); - physical_expr.operators.emplace_back(make_shared(column_view, *rpn)); - state.cur_operator = state.cur_operator + 1; + state.cur_operator += 1; } /*--------------------------------------------------------------------------------------------------------------------*\ @@ -1097,7 +1111,7 @@ void make_dec_fls_str_uncompressed_expr(PhysicalExpr& physical_expr, \*--------------------------------------------------------------------------------------------------------------------*/ template void make_dec_fsst_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>(column_view, state)); physical_expr.operators.emplace_back(make_shared(physical_expr, column_view, state)); state.cur_operator = state.cur_operator + 2; @@ -1108,7 +1122,7 @@ void make_dec_fsst_expr(PhysicalExpr& physical_expr, const ColumnView& column_vi \*--------------------------------------------------------------------------------------------------------------------*/ template void make_dec_fsst_delta_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>(column_view, state)); physical_expr.operators.emplace_back(make_shared>(physical_expr, column_view, state)); physical_expr.operators.emplace_back(make_shared(physical_expr, column_view, state)); @@ -1122,7 +1136,7 @@ template void make_dec_fsst_delta_slpatch_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>(column_view, state)); physical_expr.operators.emplace_back(make_shared>(physical_expr, column_view, state)); physical_expr.operators.emplace_back(make_shared>(physical_expr, column_view, state)); @@ -1135,7 +1149,7 @@ void make_dec_fsst_delta_slpatch_expr(PhysicalExpr& physical_expr, \*--------------------------------------------------------------------------------------------------------------------*/ template void make_dec_ffor_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>>(column_view, state)); } @@ -1144,7 +1158,7 @@ void make_dec_ffor_expr(PhysicalExpr& physical_expr, const ColumnView& column_vi \*--------------------------------------------------------------------------------------------------------------------*/ template void make_dec_ffor_slpatch_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>>(column_view, state)); physical_expr.operators.emplace_back(make_shared>(physical_expr, column_view, state)); } @@ -1154,7 +1168,6 @@ void make_dec_ffor_slpatch_expr(PhysicalExpr& physical_expr, const ColumnView& c \*--------------------------------------------------------------------------------------------------------------------*/ template void make_dec_alp_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; physical_expr.operators.emplace_back(make_shared>(column_view, state)); } @@ -1163,7 +1176,7 @@ void make_dec_alp_expr(PhysicalExpr& physical_expr, const ColumnView& column_vie \*--------------------------------------------------------------------------------------------------------------------*/ template void make_dec_galp_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>(column_view, state)); } @@ -1172,7 +1185,6 @@ void make_dec_galp_expr(PhysicalExpr& physical_expr, const ColumnView& column_vi \*--------------------------------------------------------------------------------------------------------------------*/ template void make_dec_alp_rd_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; physical_expr.operators.emplace_back(make_shared>(column_view, state)); } @@ -1181,7 +1193,7 @@ void make_dec_alp_rd_expr(PhysicalExpr& physical_expr, const ColumnView& column_ \*--------------------------------------------------------------------------------------------------------------------*/ template void make_dec_dict_ffor_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>(column_view, state)); physical_expr.operators.emplace_back( make_shared>(physical_expr, column_view, state)); @@ -1192,7 +1204,7 @@ void make_dec_dict_ffor_expr(PhysicalExpr& physical_expr, const ColumnView& colu \*--------------------------------------------------------------------------------------------------------------------*/ template void make_dec_null_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>(physical_expr, column_view, state)); } @@ -1201,7 +1213,7 @@ void make_dec_null_expr(PhysicalExpr& physical_expr, const ColumnView& column_vi \*--------------------------------------------------------------------------------------------------------------------*/ template void make_dec_frequency_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>(physical_expr, column_view, state)); } @@ -1209,7 +1221,7 @@ void make_dec_frequency_expr(PhysicalExpr& physical_expr, const ColumnView& colu * make_dec_frequency_str_expr \*--------------------------------------------------------------------------------------------------------------------*/ void make_dec_frequency_str_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared(physical_expr, column_view, state)); } @@ -1218,7 +1230,7 @@ void make_dec_frequency_str_expr(PhysicalExpr& physical_expr, const ColumnView& \*--------------------------------------------------------------------------------------------------------------------*/ template void make_dec_cross_rle_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>(physical_expr, column_view, state)); } @@ -1229,7 +1241,7 @@ template void make_dec_dict_ffor_slpatch_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>(column_view, state)); physical_expr.operators.emplace_back(make_shared>(physical_expr, column_view, state)); physical_expr.operators.emplace_back( @@ -1241,7 +1253,7 @@ void make_dec_dict_ffor_slpatch_expr(PhysicalExpr& physical_expr, \*--------------------------------------------------------------------------------------------------------------------*/ template void make_dec_fsst_dict_ffor_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>(column_view, state)); physical_expr.operators.emplace_back(make_shared(physical_expr, column_view, state)); } @@ -1253,7 +1265,7 @@ template void make_dec_fsst_dict_ffor_slpatch_expr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; physical_expr.operators.emplace_back(make_shared>(column_view, state)); physical_expr.operators.emplace_back(make_shared>(physical_expr, column_view, state)); physical_expr.operators.emplace_back(make_shared(physical_expr, column_view, state)); @@ -1267,11 +1279,12 @@ void make_dec_fsst_dict_expr(RowgroupReader& reader, PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; - auto [operator_tokens, operand_tokens] = *column_view.column_descriptor.encoding_rpn; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; + const auto* rpn = column_view.column_descriptor.encoding_rpn(); + const auto* operand_tokens = rpn->operand_tokens(); - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; - physical_expr.operators.emplace_back(reader.m_expressions[operand_tokens.at(0)]); + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; + physical_expr.operators.emplace_back(reader.m_expressions[static_cast(operand_tokens->Get(0))]); physical_expr.operators.emplace_back(make_shared(physical_expr, column_view, state)); } @@ -1283,10 +1296,10 @@ void make_dec_dict_expr(RowgroupReader& reader, PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { + const auto* operand_tokens = column_view.column_descriptor.encoding_rpn()->operand_tokens(); - auto [operator_tokens, operand_tokens] = *column_view.column_descriptor.encoding_rpn; - - physical_expr.operators.emplace_back(reader.m_expressions[operand_tokens.at(state.cur_operand++)]); + physical_expr.operators.emplace_back( + reader.m_expressions[static_cast(operand_tokens->Get(static_cast(state.cur_operand++)))]); physical_expr.operators.emplace_back( make_shared>(physical_expr, column_view, state)); } @@ -1299,10 +1312,7 @@ void make_dec_rle_expr(RowgroupReader& reader, PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - - auto [operator_tokens, operand_tokens] = *column_view.column_descriptor.encoding_rpn; - - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; state.cur_operator = 0; physical_expr.operators.emplace_back(make_shared>(column_view, state)); @@ -1319,10 +1329,7 @@ void make_dec_rle_slpatch_expr(RowgroupReader& reader, PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - - auto [operator_tokens, operand_tokens] = *column_view.column_descriptor.encoding_rpn; - - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; state.cur_operator = 0; physical_expr.operators.emplace_back(make_shared>(column_view, state)); @@ -1340,16 +1347,14 @@ void make_dec_delta_expr(RowgroupReader& reader, PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) { - - auto [operator_tokens, operand_tokens] = *column_view.column_descriptor.encoding_rpn; - - state.cur_operand = column_view.column_descriptor.encoding_rpn->operand_tokens.size() - 1; + state.cur_operand = column_view.column_descriptor.encoding_rpn()->operand_tokens()->size() - 1; state.cur_operator = 0; physical_expr.operators.emplace_back(make_shared>>(column_view, state)); physical_expr.operators.emplace_back(make_shared>(physical_expr, column_view, state)); physical_expr.operators.emplace_back(make_shared>(physical_expr, column_view, state)); } + /*--------------------------------------------------------------------------------------------------------------------*\ * make_dec_fls_str_uncompressed_expr \*--------------------------------------------------------------------------------------------------------------------*/ @@ -1381,21 +1386,24 @@ void make_dec_constant_str_expr(PhysicalExpr& physical_expr, const ColumnView& c /*--------------------------------------------------------------------------------------------------------------------*\ * make_dec_equality_expr \*--------------------------------------------------------------------------------------------------------------------*/ -void make_dec_equality_expr(PhysicalExpr& physical_expr, RowgroupReader& reader, const vector& operand_tokens) { - // - physical_expr.operators.emplace_back( - reader.m_expressions[operand_tokens.at(0)] - ->operators[reader.m_expressions[operand_tokens.at(0)]->operators.size() - 1]); +void make_dec_equality_expr(PhysicalExpr& physical_expr, + RowgroupReader& reader, + const flatbuffers::Vector* operand_tokens) { + FLS_ASSERT_NOT_NULL_POINTER(operand_tokens); + FLS_ASSERT_FB_NOT_EMPTY(operand_tokens); + + const auto idx = static_cast(operand_tokens->Get(0)); + physical_expr.operators.emplace_back(reader.m_expressions[idx]->operators.back()); } /*--------------------------------------------------------------------------------------------------------------------*\ * make_dec_struct_expr \*--------------------------------------------------------------------------------------------------------------------*/ -void make_dec_struct_expr(const ColumnDescriptorT& column_descriptor, - const ColumnView& column_view, - PhysicalExpr& physical_expr, - InterpreterState& state, - RowgroupReader& reader) { +void make_dec_struct_expr(const ColumnDescriptor& column_descriptor, + const ColumnView& column_view, + PhysicalExpr& physical_expr, + InterpreterState& state, + RowgroupReader& reader) { physical_expr.operators.emplace_back(make_shared(column_descriptor, column_view, state, reader)); state.cur_operator = state.cur_operator + 1; @@ -1404,15 +1412,24 @@ void make_dec_struct_expr(const ColumnDescriptorT& column_descriptor, /*--------------------------------------------------------------------------------------------------------------------*\ * Interpreter \*--------------------------------------------------------------------------------------------------------------------*/ -void Interpreter::Decoding::Interpret(const ColumnDescriptorT& column_descriptor, - const ColumnView& column_view, - PhysicalExpr& physical_expr, - InterpreterState& state, - RowgroupReader& reader) { +void Interpreter::Decoding::Interpret(const ColumnDescriptor& column_descriptor, + const ColumnView& column_view, + PhysicalExpr& physical_expr, + InterpreterState& state, + RowgroupReader& reader) { + const auto* rpn = column_descriptor.encoding_rpn(); + FLS_ASSERT_NOT_NULL_POINTER(rpn); + + const auto* operator_tokens = rpn->operator_tokens(); + const auto* operand_tokens = rpn->operand_tokens(); + + FLS_ASSERT_NOT_NULL_POINTER(operator_tokens); + + using enum OperatorToken; // if you already use this in the switch + + for (std::uint32_t i = 0; i < operator_tokens->size(); ++i) { + const auto operator_token = operator_tokens->Get(i); - for (const auto& [operator_tokens, operand_tokens] = *column_descriptor.encoding_rpn; - const auto& operator_token : operator_tokens) { - using enum OperatorToken; switch (operator_token) { case EXP_UNCOMPRESSED_I64: { make_dec_uncompressed_expr(physical_expr, column_view, state); @@ -2027,10 +2044,10 @@ void Interpreter::Decoding::Interpret(const ColumnDescriptorT& column_descriptor /*--------------------------------------------------------------------------------------------------------------------*\ * make_decoding_expression \*--------------------------------------------------------------------------------------------------------------------*/ -sp make_decoding_expression(const ColumnDescriptorT& column_descriptor, - const ColumnView& column_view, - RowgroupReader& reader, - InterpreterState& state) { +sp make_decoding_expression(const ColumnDescriptor& column_descriptor, + const ColumnView& column_view, + RowgroupReader& reader, + InterpreterState& state) { auto physical_expr = make_shared(); Interpreter::Decoding::Interpret(column_descriptor, column_view, *physical_expr, state, reader); diff --git a/src/expression/null_operator.cpp b/src/expression/null_operator.cpp index 30031de2..9060db53 100644 --- a/src/expression/null_operator.cpp +++ b/src/expression/null_operator.cpp @@ -75,17 +75,18 @@ template struct enc_null_opr; template struct enc_null_opr; /*--------------------------------------------------------------------------------------------------------------------*\ - * dec slpatch opr + * dec null opr \*--------------------------------------------------------------------------------------------------------------------*/ template -dec_null_opr::dec_null_opr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) - : vals_segment( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 2])) +dec_null_opr::dec_null_opr(PhysicalExpr& /*physical_expr*/, const ColumnView& column_view, InterpreterState& state) + : vals_segment(column_view.GetSegment((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 2)])) , vals_position_segment( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 1])) - , n_vals_segment( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 0])) { - state.cur_operand = state.cur_operand - 3; + column_view.GetSegment((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 1)])) + , n_vals_segment(column_view.GetSegment((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 0)])) { + state.cur_operand -= 3; } template diff --git a/src/expression/rle_expression.cpp b/src/expression/rle_expression.cpp index 84710eb7..a98c00a9 100644 --- a/src/expression/rle_expression.cpp +++ b/src/expression/rle_expression.cpp @@ -146,10 +146,11 @@ template dec_rle_map_opr::dec_rle_map_opr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) - : rle_vals_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand])) { + : rle_vals_segment_view(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand)]))) { visit(RLEExprVisitor {idxs}, physical_expr.operators.back()); - state.cur_operand = state.cur_operand - 1; + state.cur_operand -= 1; } template @@ -177,13 +178,16 @@ template dec_rle_map_opr::dec_rle_map_opr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) - : rle_vals_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 1])) - , rle_offset_segment_view( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 0])) { + : rle_vals_segment_view(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 1)]))) + , rle_offset_segment_view(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 0)]))) { visit(RLEExprVisitor {idxs}, physical_expr.operators.back()); - state.cur_operand = state.cur_operand - 2; + state.cur_operand -= 2; } + template void dec_rle_map_opr::PointTo(n_t vec_n) { rle_vals_segment_view.PointTo(vec_n); diff --git a/src/expression/slpatch_operator.cpp b/src/expression/slpatch_operator.cpp index 7c552f04..b78301e8 100644 --- a/src/expression/slpatch_operator.cpp +++ b/src/expression/slpatch_operator.cpp @@ -116,13 +116,16 @@ template dec_slpatch_opr::dec_slpatch_opr(PhysicalExpr& physical_expr, const ColumnView& column_view, InterpreterState& state) - : exceptions_segment( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 2])) - , exceptions_position_segment( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 1])) - , n_exceptions_segment( - column_view.GetSegment(column_view.column_descriptor.encoding_rpn->operand_tokens[state.cur_operand - 0])) { - state.cur_operand = state.cur_operand - 3; + : exceptions_segment(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 2)]))) + , exceptions_position_segment(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 1)]))) + , n_exceptions_segment(column_view.GetSegment( + static_cast((*column_view.column_descriptor.encoding_rpn() + ->operand_tokens())[static_cast(state.cur_operand - 0)]))) { + state.cur_operand -= 3; visit(SLPatchExprVisitor {*this}, physical_expr.operators.back()); } diff --git a/src/file/file_header.cpp b/src/file/file_header.cpp index d2b5a0f4..fa554334 100644 --- a/src/file/file_header.cpp +++ b/src/file/file_header.cpp @@ -29,7 +29,7 @@ void FileHeader::Write(const Connection& connection, const path& file_path) { } Status FileHeader::Load(FileHeader& file_header, const path& file_path) { - io io = make_unique(file_path); // todo[IO] + const io io = make_unique(file_path); // todo[IO] if (const auto file_size = IO::get_size(io); file_size < sizeof(FileHeader) + sizeof(FileFooter)) { return Status::Error(Status::ErrorCode::ERR_1_SMALL_FILE_SIZE); diff --git a/src/footer/table_descriptor.cpp b/src/footer/table_descriptor.cpp index dbfaf066..cb65b807 100644 --- a/src/footer/table_descriptor.cpp +++ b/src/footer/table_descriptor.cpp @@ -4,111 +4,163 @@ // src/footer/table_descriptor.cpp // ──────────────────────────────────────────────────────── #include "fls/footer/table_descriptor.hpp" +#include "flatbuffers/flatbuffer_builder.h" +#include "flatbuffers/verifier.h" // flatbuffers::Verifier #include "fls/common/alias.hpp" #include "fls/cor/lyt/buf.hpp" -#include "fls/flatbuffers/flatbuffers.hpp" #include "fls/footer/rowgroup_descriptor.hpp" #include "fls/footer/table_descriptor_generated.h" #include "fls/io/file.hpp" -#include "fls/json/fls_json.hpp" #include "fls/std/filesystem.hpp" +#include "fls/std/vector.hpp" #include "fls/table/table.hpp" -#include // for uint8_t, n_t -#include // for std::ifstream -#include // for std::ios flags like std::ios::binary, std::ios::ate -#include // for std::runtime_error -#include // for std::vector +#include // std::size_t +#include // uint8_t +#include // std::memcpy +#include +#include +#include // std::shared_ptr, std::make_shared +#include +#include // std::move +#include namespace fastlanes { -// TableDescriptor::TableDescriptor() -// : m_table_binary_size {0} { -// } -// -// // Deep-copy: clone each RowgroupDescriptor via its copy-ctor -// TableDescriptor::TableDescriptor(const TableDescriptor& o) -// : m_rowgroup_descriptors {} // start empty -// , m_table_binary_size(o.m_table_binary_size) { -// m_rowgroup_descriptors.reserve(o.m_rowgroup_descriptors.size()); -// for (auto const& up_rg : o.m_rowgroup_descriptors) { -// // invoke RowgroupDescriptor’s own copy-ctor -// m_rowgroup_descriptors.emplace_back(std::make_unique(*up_rg)); -// } -// } -// -// n_t TableDescriptor::GetNRowgroups() const { -// return m_rowgroup_descriptors.size(); -// } - -// -// template -// up make_table_descriptor(const path& file_path) { -// auto json_string = File::read(file_path); -// const nlohmann::json j = nlohmann::json::parse(json_string); -// auto table_descriptor = j.get(); -// return make_unique(table_descriptor); -// } - -up make_table_descriptor(const Table& table) { - auto table_descriptor = make_unique(); - - for (n_t rowgroup_idx = 0; rowgroup_idx < table.get_n_rowgroups(); ++rowgroup_idx) { - table_descriptor->m_rowgroup_descriptors.push_back(make_rowgroup_descriptor(*table.m_rowgroups[rowgroup_idx])); +const TableDescriptor& get_table_descriptor(const uint8_t* data, std::size_t size) { + if (!data || size == 0) { + throw std::invalid_argument("get_table_descriptor: null data or zero size"); } - table_descriptor->m_table_binary_size = 0; - - return table_descriptor; +#if defined(DEBUG) + { + flatbuffers::Verifier v(data, size); + if (!VerifyTableDescriptorBuffer(v)) { + throw std::runtime_error("get_table_descriptor: invalid TableDescriptor FlatBuffer"); + } + } +#endif + return *GetTableDescriptor(data); } -up make_table_descriptor(const path& file_path) { - // init +const TableDescriptor& make_table_descriptor(const path& file_path, std::vector& storage) { std::ifstream in {file_path, std::ios::binary | std::ios::ate}; if (!in) { - throw std::runtime_error("Failed to open footer: " + file_path.string()); + throw std::runtime_error("make_table_descriptor: failed to open footer: " + file_path.string()); + } + + const auto sz = static_cast(in.tellg()); + if (sz == 0) { + throw std::runtime_error("make_table_descriptor: empty footer file: " + file_path.string()); } - // - auto size = in.tellg(); + storage.resize(sz); in.seekg(0, std::ios::beg); - std::vector buffer(static_cast(size)); - if (!in.read(reinterpret_cast(buffer.data()), size)) { - throw std::runtime_error("Failed to read footer: " + file_path.string()); + if (!in.read(reinterpret_cast(storage.data()), static_cast(sz))) { + throw std::runtime_error("make_table_descriptor: failed to read footer: " + file_path.string()); + } + + return get_table_descriptor(storage.data(), storage.size()); +} + +const TableDescriptor& +make_table_descriptor(const path& file_path, n_t offset, n_t size, std::vector& storage) { + File f(file_path); + Buf buf; + + f.ReadRange(buf, offset, size); + + const auto* p = reinterpret_cast(buf.data()); + storage.assign(p, p + buf.Size()); + + return get_table_descriptor(storage.data(), storage.size()); +} + +const TableDescriptor* TableDescriptorHandle::MakePtr(const std::shared_ptr>& bytes, bool verify) { + if (!bytes || bytes->empty()) { + throw std::runtime_error("TableDescriptorHandle: empty buffer"); } - in.close(); -#ifdef DEBUG - flatbuffers::Verifier verifier(buffer.data(), buffer.size()); - if (!fastlanes::VerifyTableDescriptorBuffer(verifier)) { - throw std::runtime_error("Invalid FlatBuffer in file: " + file_path.string()); + const uint8_t* data = bytes->data(); + const std::size_t sz = bytes->size(); + +#if defined(DEBUG) + (void)verify; // always verify in DEBUG + flatbuffers::Verifier v(data, sz); + if (!VerifyTableDescriptorBuffer(v)) { + throw std::runtime_error("TableDescriptorHandle: verification failed (DEBUG)"); + } +#else + if (verify) { + flatbuffers::Verifier v(data, sz); + if (!VerifyTableDescriptorBuffer(v)) { + throw std::runtime_error("TableDescriptorHandle: verification failed"); + } } #endif - const auto accessor = GetTableDescriptor(buffer.data()); + return GetTableDescriptor(data); +} + +TableDescriptorHandle TableDescriptorHandle::FromBytes(vector bytes, bool verify) { + auto sp = std::make_shared>(std::move(bytes)); + auto ptr = MakePtr(sp, verify); + return {std::move(sp), ptr}; +} + +TableDescriptorHandle TableDescriptorHandle::FromFile(const path& file_path, bool verify) { + std::ifstream in {file_path, std::ios::binary | std::ios::ate}; + if (!in) { + throw std::runtime_error("TableDescriptorHandle::FromFile: failed to open: " + file_path.string()); + } + + const auto sz = static_cast(in.tellg()); + if (sz == 0) { + throw std::runtime_error("TableDescriptorHandle::FromFile: empty file: " + file_path.string()); + } - // (deep copy) - TableDescriptorT* raw = accessor->UnPack(); - if (!raw) { - throw std::runtime_error("Failed to unpack TableDescriptor from buffer: " + file_path.string()); + vector storage(sz); + in.seekg(0, std::ios::beg); + if (!in.read(reinterpret_cast(storage.data()), static_cast(sz))) { + throw std::runtime_error("TableDescriptorHandle::FromFile: failed to read: " + file_path.string()); } - return up(raw); + return FromBytes(std::move(storage), verify); +} + +TableDescriptorHandle TableDescriptorHandle::FromFileSlice(const path& file_path, n_t offset, n_t size, bool verify) { + File f(file_path); + Buf buf; + f.ReadRange(buf, offset, size); + + const auto* p = reinterpret_cast(buf.data()); + vector storage; + storage.assign(p, p + size); + + return FromBytes(std::move(storage), verify); } -up make_table_descriptor(const path& file_path, const n_t offset, const n_t size) { - // init - File file(file_path); - Buf buf; // OPTIMIZE +TableDescriptorHandle TableDescriptorHandle::FromNative(const TableDescriptorT& native) { + flatbuffers::FlatBufferBuilder fbb; + auto off = TableDescriptor::Pack(fbb, &native); + fbb.Finish(off); + + auto det = fbb.Release(); + vector bytes(det.size()); + if (det.size() > 0) { + std::memcpy(bytes.data(), det.data(), det.size()); + } + + return FromBytes(std::move(bytes), /*verify=*/false); +} - file.ReadRange(buf, offset, size); - const auto accessor = fastlanes::GetTableDescriptor(buf.data()); +up make_table_descriptor(const Table& table) { + auto table_descriptor = make_unique(); - // 4) Deep-copy into a TableDescriptorT and wrap in a unique_ptr - TableDescriptorT* raw = accessor->UnPack(); - if (!raw) { - throw std::runtime_error("Failed to unpack TableDescriptor from slice in file: " + file_path.string()); + for (n_t rowgroup_idx = 0; rowgroup_idx < table.get_n_rowgroups(); ++rowgroup_idx) { + table_descriptor->m_rowgroup_descriptors.push_back(make_rowgroup_descriptor(*table.m_rowgroups[rowgroup_idx])); } + table_descriptor->m_table_binary_size = 0; - return up(raw); + return table_descriptor; } } // namespace fastlanes diff --git a/src/include/fls/common/assert.hpp b/src/include/fls/common/assert.hpp index e904c648..7f1b24b4 100644 --- a/src/include/fls/common/assert.hpp +++ b/src/include/fls/common/assert.hpp @@ -53,6 +53,13 @@ #define FLS_ASSERT_CORRECT_IDX(Expr) FLS_ASSERT(Expr != INVALID_N, " ", fastlanes::Assert::IDX); #define FLS_ASSERT_NOT_EMPTY_VEC(VEC) FLS_ASSERT(!VEC.empty(), " ", fastlanes::Assert::EMPTY_VECTOR); #define FLS_ASSERT_CORRECT_VEC_INDEX(C) FLS_ASSERT(C <= 1024 && C >= 0, " ", fastlanes::Assert::VEC_INDEX); +// ── FlatBuffers helpers ────────────────────────────────────────────────────── +#define FLS_ASSERT_FB_PTR(PTR) FLS_ASSERT((PTR) != nullptr, " ", fastlanes::Assert::NULL_POINTER) +#define FLS_ASSERT_FB_NOT_EMPTY(PTR) \ + FLS_ASSERT(((PTR) != nullptr) && !((PTR)->empty()), " ", fastlanes::Assert::EMPTY_VECTOR) +#define FLS_ASSERT_FB_IDX_IN_RANGE(PTR, IDX) \ + FLS_ASSERT( \ + ((PTR) != nullptr) && (static_cast(IDX) < (PTR)->size()), " ", fastlanes::Assert::OUT_OF_RANGE_INDEX) #include diff --git a/src/include/fls/expression/decoding_operator.hpp b/src/include/fls/expression/decoding_operator.hpp index 7ee92841..6317bb2e 100644 --- a/src/include/fls/expression/decoding_operator.hpp +++ b/src/include/fls/expression/decoding_operator.hpp @@ -52,7 +52,7 @@ struct dec_uncompressed_opr { * dec_uncompressed_opr \*--------------------------------------------------------------------------------------------------------------------*/ struct dec_fls_str_uncompressed_opr { - explicit dec_fls_str_uncompressed_opr(const ColumnView& column_view, const RPNT& rpn); + explicit dec_fls_str_uncompressed_opr(const ColumnView& column_view, const RPN& rpn); public: void PointTo(n_t vec_idx); @@ -78,10 +78,10 @@ struct dec_constant_str_opr { }; struct dec_struct_opr { - dec_struct_opr(const ColumnDescriptorT& column_descriptor, - const ColumnView& column_view, - InterpreterState& state, - RowgroupReader& reader); + dec_struct_opr(const ColumnDescriptor& column_descriptor, + const ColumnView& column_view, + InterpreterState& state, + RowgroupReader& reader); vector> internal_exprs; }; diff --git a/src/include/fls/expression/interpreter.hpp b/src/include/fls/expression/interpreter.hpp index 846b7612..6a19b95d 100644 --- a/src/include/fls/expression/interpreter.hpp +++ b/src/include/fls/expression/interpreter.hpp @@ -37,18 +37,18 @@ class Interpreter { }; class Decoding { public: - static void Interpret(const ColumnDescriptorT& column_descriptor, - const ColumnView& column_view, - PhysicalExpr& physical_expr, - InterpreterState& state, - RowgroupReader& reader); + static void Interpret(const ColumnDescriptor& column_descriptor, + const ColumnView& column_view, + PhysicalExpr& physical_expr, + InterpreterState& state, + RowgroupReader& reader); }; }; -sp make_decoding_expression(const ColumnDescriptorT& column_descriptor, - const ColumnView& column_view, - RowgroupReader& reader, - InterpreterState& state); +sp make_decoding_expression(const ColumnDescriptor& column_descriptor, + const ColumnView& column_view, + RowgroupReader& reader, + InterpreterState& state); } // namespace fastlanes #endif // FLS_EXPRESSION_INTERPRETER_HPP diff --git a/src/include/fls/footer/table_descriptor.hpp b/src/include/fls/footer/table_descriptor.hpp index 86566fa6..59b932c4 100644 --- a/src/include/fls/footer/table_descriptor.hpp +++ b/src/include/fls/footer/table_descriptor.hpp @@ -6,18 +6,93 @@ #ifndef FLS_FOOTER_TABLE_DESCRIPTOR_HPP #define FLS_FOOTER_TABLE_DESCRIPTOR_HPP -#include "fls/footer/rowgroup_descriptor.hpp" +#include "fls/common/alias.hpp" // n_t, up #include "fls/footer/table_descriptor_generated.h" +#include "fls/std/filesystem.hpp" // path +#include "fls/std/vector.hpp" // fastlanes::vector alias +#include +#include +#include // shared_ptr, std::make_unique namespace fastlanes { -/*--------------------------------------------------------------------------------------------------------------------*/ + class Table; -/*--------------------------------------------------------------------------------------------------------------------*/ + +class TableDescriptorHandle { +public: + TableDescriptorHandle() = default; + + // Build from already-owned bytes (moves bytes into the handle). + static TableDescriptorHandle FromBytes(vector bytes, bool verify = true); + + // Read whole file and own the bytes. + static TableDescriptorHandle FromFile(const path& file_path, bool verify = true); + + // Read a slice [offset, offset+size) from file and own the bytes. + static TableDescriptorHandle FromFileSlice(const path& file_path, n_t offset, n_t size, bool verify = true); + + // Pack native T -> bytes and expose a view. + static TableDescriptorHandle FromNative(const TableDescriptorT& native); + + // View access + [[nodiscard]] const TableDescriptor* Get() const noexcept { + return ptr_; + } + const TableDescriptor& operator*() const { + return *ptr_; + } + const TableDescriptor* operator->() const { + return ptr_; + } + [[nodiscard]] const uint8_t* data() const noexcept { + return bytes_ ? bytes_->data() : nullptr; + } + [[nodiscard]] std::size_t size() const noexcept { + return bytes_ ? bytes_->size() : 0; + } + explicit operator bool() const noexcept { + return ptr_ != nullptr; + } + + // Convert to native (owning) structure when you need mutability / STL containers. + [[nodiscard]] up Unpack() const { + if (!ptr_) { + return {}; + } + return up(ptr_->UnPack()); + } + +private: + // Internal constructor used by factory methods. + TableDescriptorHandle(sp> bytes, const TableDescriptor* ptr) + : bytes_(std::move(bytes)) + , ptr_(ptr) { + } + + // Create a typed view into the owned bytes; may verify if requested. + static const TableDescriptor* MakePtr(const sp>& bytes, bool verify); + +private: + sp> bytes_; // owning storage + const TableDescriptor* ptr_ = nullptr; // view into bytes_->data() +}; + +/*──────────────────────────────────────────────────────────────────────────────┐ +│ By-pointer helpers (return up<...>) │ +│ These heap-allocate the handle and return unique ownership (up<>). │ +└──────────────────────────────────────────────────────────────────────────────*/ up make_table_descriptor(const Table& table); -// -up make_table_descriptor(const path& file_path); -// -up make_table_descriptor(const path& file_path, n_t offset, n_t size); + +inline up make_table_descriptor(const path& file_path, bool verify = true) { + return std::make_unique(TableDescriptorHandle::FromFile(file_path, verify)); +} + +inline up +make_table_descriptor(const path& file_path, n_t offset, n_t size, bool verify = true) { + + return std::make_unique( + TableDescriptorHandle::FromFileSlice(file_path, offset, size, verify)); +} } // namespace fastlanes diff --git a/src/include/fls/reader/column_view.hpp b/src/include/fls/reader/column_view.hpp index 60ec7d62..004dcafd 100644 --- a/src/include/fls/reader/column_view.hpp +++ b/src/include/fls/reader/column_view.hpp @@ -12,8 +12,8 @@ namespace fastlanes { /*--------------------------------------------------------------------------------------------------------------------*/ -struct ColumnDescriptorT; -struct RowgroupDescriptorT; +struct ColumnDescriptor; +struct RowgroupDescriptor; class SegmentView; /*--------------------------------------------------------------------------------------------------------------------*/ /*--------------------------------------------------------------------------------------------------------------------*\ @@ -21,15 +21,15 @@ class SegmentView; \*--------------------------------------------------------------------------------------------------------------------*/ class ColumnView { public: - explicit ColumnView(span column_span, - const ColumnDescriptorT& column_descriptor, - const RowgroupDescriptorT& rowgroup_descriptor); + explicit ColumnView(span column_span, + const ColumnDescriptor& column_descriptor, + const RowgroupDescriptor& rowgroup_descriptor); [[nodiscard]] SegmentView GetSegment(n_t segment_idx) const; public: - span column_span; - const ColumnDescriptorT& column_descriptor; - vector children; + span column_span; + const ColumnDescriptor& column_descriptor; + vector children; }; } // namespace fastlanes diff --git a/src/include/fls/reader/rowgroup_reader.hpp b/src/include/fls/reader/rowgroup_reader.hpp index 2882aa46..aca706d0 100644 --- a/src/include/fls/reader/rowgroup_reader.hpp +++ b/src/include/fls/reader/rowgroup_reader.hpp @@ -22,7 +22,7 @@ class Rowgroup; /*--------------------------------------------------------------------------------------------------------------------*/ class RowgroupReader { public: - explicit RowgroupReader(const path& file_path, const RowgroupDescriptorT& rowgroup_descriptor, Connection& fls); + explicit RowgroupReader(const path& file_path, const RowgroupDescriptor& rowgroup_descriptor, Connection& fls); public: vector>& get_chunk(n_t vec_idx); @@ -33,7 +33,7 @@ class RowgroupReader { /// void to_csv(const path& dir_path); /// - [[nodiscard]] const RowgroupDescriptorT& get_descriptor() const; + [[nodiscard]] const RowgroupDescriptor& get_descriptor() const; ///! [[nodiscard]] vector get_column_names() const; /// @@ -43,10 +43,10 @@ class RowgroupReader { vector> m_expressions; private: - Connection& m_connection; - const RowgroupDescriptorT& m_rowgroup_descriptor; - up m_buf; - up m_rowgroup_view; + Connection& m_connection; + const RowgroupDescriptor& m_rowgroup_descriptor; + up m_buf; + up m_rowgroup_view; }; } // namespace fastlanes diff --git a/src/include/fls/reader/rowgroup_view.hpp b/src/include/fls/reader/rowgroup_view.hpp index 40f1a2ba..ea5ca5f0 100644 --- a/src/include/fls/reader/rowgroup_view.hpp +++ b/src/include/fls/reader/rowgroup_view.hpp @@ -11,13 +11,13 @@ namespace fastlanes { /*--------------------------------------------------------------------------------------------------------------------*/ -struct RowgroupDescriptorT; +struct RowgroupDescriptor; class ColumnView; /*--------------------------------------------------------------------------------------------------------------------*/ class RowgroupView { public: - explicit RowgroupView(span ptr, const RowgroupDescriptorT& footer); + explicit RowgroupView(span ptr, const RowgroupDescriptor& footer); public: ColumnView& operator[](n_t col_idx); diff --git a/src/include/fls/reader/segment.hpp b/src/include/fls/reader/segment.hpp index 4e9abc2e..efd4e9e1 100644 --- a/src/include/fls/reader/segment.hpp +++ b/src/include/fls/reader/segment.hpp @@ -14,7 +14,9 @@ namespace fastlanes { /*--------------------------------------------------------------------------------------------------------------------*/ class Buf; +struct SegmentDescriptor; struct SegmentDescriptorT; + /*--------------------------------------------------------------------------------------------------------------------*/ /*--------------------------------------------------------------------------------------------------------------------*\ @@ -63,7 +65,7 @@ class SegmentView { /*--------------------------------------------------------------------------------------------------------------------*\ * make_segment_view \*--------------------------------------------------------------------------------------------------------------------*/ -SegmentView make_segment_view(span column_span, const SegmentDescriptorT& segment_descriptor); +SegmentView make_segment_view(span column_span, const SegmentDescriptor& segment_descriptor); /*--------------------------------------------------------------------------------------------------------------------*\ * Segment diff --git a/src/include/fls/reader/table_reader.hpp b/src/include/fls/reader/table_reader.hpp index 06cf33a6..7ef0ae73 100644 --- a/src/include/fls/reader/table_reader.hpp +++ b/src/include/fls/reader/table_reader.hpp @@ -15,7 +15,7 @@ namespace fastlanes { /*--------------------------------------------------------------------------------------------------------------------*/ class Connection; class RowgroupReader; -struct TableDescriptorT; +class TableDescriptorHandle; class Table; /*--------------------------------------------------------------------------------------------------------------------*/ class FLS_API TableReader { @@ -35,9 +35,9 @@ class FLS_API TableReader { void to_csv(const char* file_path) const; private: - up m_table_descriptor; - Connection& m_connection; - const path m_file_path; + up m_table_descriptor_handle; + Connection& m_connection; + const path m_file_path; }; } // namespace fastlanes diff --git a/src/reader/column_view.cpp b/src/reader/column_view.cpp index 922e01ad..38a5ec1d 100644 --- a/src/reader/column_view.cpp +++ b/src/reader/column_view.cpp @@ -10,25 +10,33 @@ #include "fls/reader/segment.hpp" #include "fls/std/span.hpp" #include // for std::byte +#include // for std::uint32_t namespace fastlanes { -ColumnView::ColumnView(const span column_span, - const ColumnDescriptorT& column_descriptor, - const RowgroupDescriptorT& rowgroup_descriptor) +ColumnView::ColumnView(const span column_span, + const ColumnDescriptor& column_descriptor, + const RowgroupDescriptor& rowgroup_descriptor) : column_span(column_span) , column_descriptor(column_descriptor) { - for (n_t child_col_idx {0}; child_col_idx < column_descriptor.children.size(); ++child_col_idx) { - auto& child_column_descriptor = column_descriptor.children[child_col_idx]; - children.emplace_back(column_span, *child_column_descriptor, rowgroup_descriptor); + if (!column_descriptor.children()) { + return; + } + FLS_ASSERT_NOT_NULL_POINTER(column_descriptor.children()) + + for (n_t child_col_idx {0}; child_col_idx < column_descriptor.children()->size(); ++child_col_idx) { + + auto& child_column_descriptor = *(*column_descriptor.children())[static_cast(child_col_idx)]; + children.emplace_back(column_span, child_column_descriptor, rowgroup_descriptor); } } SegmentView ColumnView::GetSegment(n_t segment_idx) const { - FLS_ASSERT_L(segment_idx, column_descriptor.segment_descriptors.size()); + FLS_ASSERT_L(segment_idx, column_descriptor.segment_descriptors()->size()); - return make_segment_view(column_span, *column_descriptor.segment_descriptors[segment_idx]); + return make_segment_view(column_span, + *(*column_descriptor.segment_descriptors())[static_cast(segment_idx)]); } } // namespace fastlanes diff --git a/src/reader/rowgroup_reader.cpp b/src/reader/rowgroup_reader.cpp index 4a6cf680..5caa9a94 100644 --- a/src/reader/rowgroup_reader.cpp +++ b/src/reader/rowgroup_reader.cpp @@ -21,34 +21,35 @@ #include "fls/reader/rowgroup_view.hpp" #include "fls/std/filesystem.hpp" #include "fls/table/chunk.hpp" // for Chunk +#include // for std::uint32_t #include // for make_unique, uniqu... namespace fastlanes { -RowgroupReader::RowgroupReader(const path& file_path, - const RowgroupDescriptorT& rowgroup_descriptor, - Connection& connection) +RowgroupReader::RowgroupReader(const path& file_path, + const RowgroupDescriptor& rowgroup_descriptor, + Connection& connection) : m_connection(connection) , m_rowgroup_descriptor(rowgroup_descriptor) { // read file { // allocate buffer - m_buf = make_unique(m_rowgroup_descriptor.m_size); // todo[memory_pool] - io io = make_unique(file_path); // todo[IO] - IO::range_read(io, *m_buf, m_rowgroup_descriptor.m_offset, m_rowgroup_descriptor.m_size); + m_buf = make_unique(m_rowgroup_descriptor.m_size()); // todo[memory_pool] + io io = make_unique(file_path); // todo[IO] + IO::range_read(io, *m_buf, m_rowgroup_descriptor.m_offset(), m_rowgroup_descriptor.m_size()); m_rowgroup_view = make_unique(m_buf->Span(), m_rowgroup_descriptor); } // init level 1 expression { - m_expressions.reserve(m_rowgroup_descriptor.m_column_descriptors.size()); - for (n_t col_idx {0}; col_idx < m_rowgroup_descriptor.m_column_descriptors.size(); ++col_idx) { - auto& column_descriptor = m_rowgroup_descriptor.m_column_descriptors[col_idx]; + m_expressions.reserve(m_rowgroup_descriptor.m_column_descriptors()->size()); + for (n_t col_idx {0}; col_idx < m_rowgroup_descriptor.m_column_descriptors()->size(); ++col_idx) { + auto& column_descriptor = *(*m_rowgroup_descriptor.m_column_descriptors())[static_cast(col_idx)]; auto& column_view = (*m_rowgroup_view)[col_idx]; InterpreterState state; - auto physical_expr = make_decoding_expression(*column_descriptor, column_view, *this, state); + auto physical_expr = make_decoding_expression(column_descriptor, column_view, *this, state); ExprExecutor::CountOperator(*physical_expr); m_expressions.emplace_back(physical_expr); } @@ -56,7 +57,7 @@ RowgroupReader::RowgroupReader(const path& file_path, } vector>& RowgroupReader::get_chunk(const n_t vec_idx) { - for (n_t col_idx {0}; col_idx < m_rowgroup_descriptor.m_column_descriptors.size(); ++col_idx) { + for (n_t col_idx {0}; col_idx < m_rowgroup_descriptor.m_column_descriptors()->size(); ++col_idx) { auto& physical_expr = *m_expressions[col_idx]; ExprExecutor::smart_execute(physical_expr, vec_idx); } @@ -66,18 +67,24 @@ vector>& RowgroupReader::get_chunk(const n_t vec_idx) { void RowgroupReader::reset() { } -const RowgroupDescriptorT& RowgroupReader::get_descriptor() const { +const RowgroupDescriptor& RowgroupReader::get_descriptor() const { return m_rowgroup_descriptor; } up RowgroupReader::materialize() { - auto rowgroup_up = std::make_unique(m_rowgroup_descriptor, m_connection); + // Convert FlatBuffers table -> native T + auto rg_native = up(m_rowgroup_descriptor.UnPack()); + + // Construct Rowgroup from the native descriptor + auto rowgroup_up = std::make_unique(*rg_native, m_connection); + const Materializer materializer {*rowgroup_up}; - for (n_t vec_idx {0}; vec_idx < m_rowgroup_descriptor.m_n_vec; vec_idx++) { + const n_t n_vec = static_cast(m_rowgroup_descriptor.m_n_vec()); + for (n_t vec_idx {0}; vec_idx < n_vec; ++vec_idx) { auto& expressions = get_chunk(vec_idx); materializer.Materialize(expressions, vec_idx); - }; + } // materializer.rowgroup.Cast(); materializer.rowgroup.Finalize(); diff --git a/src/reader/rowgroup_view.cpp b/src/reader/rowgroup_view.cpp index d33a2c8b..d5704fbf 100644 --- a/src/reader/rowgroup_view.cpp +++ b/src/reader/rowgroup_view.cpp @@ -13,9 +13,9 @@ namespace fastlanes { -RowgroupView::RowgroupView(span ptr, const RowgroupDescriptorT& footer) { +RowgroupView::RowgroupView(span ptr, const RowgroupDescriptor& footer) { - for (const auto& column_descriptor : footer.m_column_descriptors) { + for (const auto& column_descriptor : *footer.m_column_descriptors()) { const span column_span = ptr; columns.emplace_back(make_unique(column_span, *column_descriptor, footer)); } diff --git a/src/reader/segment.cpp b/src/reader/segment.cpp index 0497ad65..005c02bf 100644 --- a/src/reader/segment.cpp +++ b/src/reader/segment.cpp @@ -235,17 +235,18 @@ template u08_pt* Segment::GetFixedSizeArray(n_t length); /*--------------------------------------------------------------------------------------------------------------------*\ * make_segment_view \*--------------------------------------------------------------------------------------------------------------------*/ -SegmentView make_segment_view(span column_span, const SegmentDescriptorT& segment_descriptor) { - auto segment_span = column_span.subspan(segment_descriptor.entrypoint_offset, segment_descriptor.entrypoint_size); +SegmentView make_segment_view(span column_span, const SegmentDescriptor& segment_descriptor) { + auto segment_span = + column_span.subspan(segment_descriptor.entrypoint_offset(), segment_descriptor.entrypoint_size()); - switch (segment_descriptor.entry_point_t) { + switch (segment_descriptor.entry_point_t()) { case EntryPointType::UINT8: { const auto entry_point_span = std::span(reinterpret_cast(segment_span.data()), segment_span.size() / sizeof(uint8_t)); auto entry_point_view = EntryPointView(entry_point_span); - const auto data_span = column_span.subspan(segment_descriptor.data_offset, segment_descriptor.data_size); + const auto data_span = column_span.subspan(segment_descriptor.data_offset(), segment_descriptor.data_size()); return SegmentView {entry_point_view, data_span}; } case EntryPointType::UINT16: { @@ -254,7 +255,7 @@ SegmentView make_segment_view(span column_span, const SegmentDescript auto entry_point_view = EntryPointView(entry_point_span); - const auto data_span = column_span.subspan(segment_descriptor.data_offset, segment_descriptor.data_size); + const auto data_span = column_span.subspan(segment_descriptor.data_offset(), segment_descriptor.data_size()); return SegmentView {entry_point_view, data_span}; } case EntryPointType::UINT32: { @@ -263,7 +264,7 @@ SegmentView make_segment_view(span column_span, const SegmentDescript auto entry_point_view = EntryPointView(entry_point_span); - const auto data_span = column_span.subspan(segment_descriptor.data_offset, segment_descriptor.data_size); + const auto data_span = column_span.subspan(segment_descriptor.data_offset(), segment_descriptor.data_size()); return SegmentView {entry_point_view, data_span}; } case EntryPointType::UINT64: diff --git a/src/reader/table_reader.cpp b/src/reader/table_reader.cpp index 9912af3e..8170dd0a 100644 --- a/src/reader/table_reader.cpp +++ b/src/reader/table_reader.cpp @@ -10,27 +10,33 @@ #include "fls/file/file_footer.hpp" #include "fls/file/file_header.hpp" #include "fls/footer/table_descriptor.hpp" +#include "fls/footer/table_descriptor_generated.h" #include "fls/reader/rowgroup_reader.hpp" #include "fls/std/filesystem.hpp" #include "fls/std/string.hpp" -#include // std::filesystem::path, exists, is_directory, is_regular_file -#include // for std::move +#include // std::filesystem::path, exists, is_directory, is_regular_file +#include // flatbuffers::uoffset_t +#include // std::move namespace fastlanes { constexpr static auto const* TABLE_DESCRIPTOR_FILE_NAME {"table_descriptor.fbb"}; up TableReader::get_rowgroup_reader(const n_t rowgroup_idx) const { - auto rowgroup_reader = make_unique( - m_file_path, *m_table_descriptor->m_rowgroup_descriptors[rowgroup_idx], m_connection); - return rowgroup_reader; + const TableDescriptor* td = m_table_descriptor_handle->Get(); + const auto fb_idx = static_cast(rowgroup_idx); + const auto* rg = td->m_rowgroup_descriptors()->Get(fb_idx); // pointer to RowgroupDescriptor (table) + return make_unique(m_file_path, *rg, m_connection); } up TableReader::materialize() const { auto table_up = make_unique
(m_connection); - for (n_t rowgroup_idx {0}; rowgroup_idx < m_table_descriptor->m_rowgroup_descriptors.size(); rowgroup_idx++) { - auto rowgroup_up = get_rowgroup_reader(rowgroup_idx)->materialize(); + const TableDescriptor* td = m_table_descriptor_handle->Get(); + const auto n_rgs = td->m_rowgroup_descriptors()->size(); // uoffset_t + + for (flatbuffers::uoffset_t i = 0; i < n_rgs; ++i) { + auto rowgroup_up = get_rowgroup_reader(static_cast(i))->materialize(); table_up->m_rowgroups.push_back(std::move(rowgroup_up)); } @@ -38,8 +44,11 @@ up
TableReader::materialize() const { } void TableReader::to_csv(const path& file_path) const { - for (n_t rowgroup_idx {0}; rowgroup_idx < m_table_descriptor->m_rowgroup_descriptors.size(); rowgroup_idx++) { - auto rowgroup_up = get_rowgroup_reader(rowgroup_idx)->materialize(); + const TableDescriptor* td = m_table_descriptor_handle->Get(); + const auto n_rgs = td->m_rowgroup_descriptors()->size(); + + for (flatbuffers::uoffset_t i = 0; i < n_rgs; ++i) { + auto rowgroup_up = get_rowgroup_reader(static_cast(i))->materialize(); CSV::to_csv(file_path, *rowgroup_up, rowgroup_up->m_descriptor); } } @@ -55,7 +64,6 @@ void TableReader::to_csv(const char* file_path) const { TableReader::TableReader(const path& file_path, Connection& connection) : m_connection(connection) , m_file_path(file_path) { - FileFooter file_footer {}; FileHeader file_header {}; @@ -63,17 +71,18 @@ TableReader::TableReader(const path& file_path, Connection& connection) FileFooter::Load(file_footer, file_path); if (file_header.settings.inline_footer) { - m_table_descriptor = + // Inline footer: read slice + m_table_descriptor_handle = make_table_descriptor(file_path, file_footer.table_descriptor_offset, file_footer.table_descriptor_size); } else { - - m_table_descriptor = make_table_descriptor(file_path.parent_path() / TABLE_DESCRIPTOR_FILE_NAME); + // External footer file + m_table_descriptor_handle = make_table_descriptor(file_path.parent_path() / TABLE_DESCRIPTOR_FILE_NAME); } } + up TableReader::operator[](const n_t rowgroup_idx) const { - auto rowgroup_reader = make_unique( - m_file_path, *m_table_descriptor->m_rowgroup_descriptors[rowgroup_idx], m_connection); - return rowgroup_reader; + // Delegate to the helper to keep logic in one place + return get_rowgroup_reader(rowgroup_idx); } } // namespace fastlanes diff --git a/src/table/rowgroup.cpp b/src/table/rowgroup.cpp index 93b09a98..9ab110de 100644 --- a/src/table/rowgroup.cpp +++ b/src/table/rowgroup.cpp @@ -108,9 +108,8 @@ void init_logical_columns(const ColumnDescriptors& footer, rowgroup_pt& columns) columns.emplace_back(init_logical_columns(*col_descriptor)); } } - Rowgroup::Rowgroup(const RowgroupDescriptorT& footer, const Connection& connection) - : m_descriptor(footer) + : m_descriptor(footer) // RowgroupDescriptor -> RowgroupDescriptorT , n_tup(footer.m_n_tuples) , m_connection(connection) , capacity(connection.m_config->n_vector_per_rowgroup * CFG::VEC_SZ) { diff --git a/test/include/fls_tester.hpp b/test/include/fls_tester.hpp index aa73f30e..71fe30f4 100644 --- a/test/include/fls_tester.hpp +++ b/test/include/fls_tester.hpp @@ -10,15 +10,18 @@ #include "fastlanes.hpp" #include "fls/connection.hpp" #include "gtest/gtest.h" +#include // std::filesystem::exists, create_directories, remove_all #include + #if defined(_WIN32) -#include // for _getpid() +#include // _getpid #define getpid _getpid #else -#include // for getpid() +#include // getpid #endif namespace fastlanes { + class FastLanesReaderTester : public ::testing::Test { const path fastlanes_repo_data_path {FLS_CMAKE_SOURCE_DIR}; const path fls_dir_path = fastlanes_repo_data_path / "data" / "fls"; @@ -28,14 +31,14 @@ class FastLanesReaderTester : public ::testing::Test { void SetUp() override { fls_dir_path_process_specific = fls_dir_path / std::to_string(getpid()); fls_file_path_process_specific = fls_dir_path_process_specific / "data.fls"; - if (!exists(fls_dir_path_process_specific)) { - create_directories(fls_dir_path_process_specific); + if (!std::filesystem::exists(fls_dir_path_process_specific)) { + std::filesystem::create_directories(fls_dir_path_process_specific); } }; void TearDown() override { - if (exists(fls_dir_path_process_specific)) { - remove_all(fls_dir_path_process_specific); + if (std::filesystem::exists(fls_dir_path_process_specific)) { + std::filesystem::remove_all(fls_dir_path_process_specific); } } @@ -45,42 +48,36 @@ class FastLanesReaderTester : public ::testing::Test { const vector& expressions = {}, const n_t rowgroup_size = CFG::RowGroup::N_VECTORS_PER_ROWGROUP, bool inlined_footer = false) const { - const path dir_path = fastlanes_repo_data_path / string(table); - // original rowgroup - Connection con1; + // original table + Connection con1; con1.reset().set_n_vectors_per_rowgroup(rowgroup_size); if (inlined_footer) { con1.inline_footer(); } - if (!expressions.empty()) { con1.force_schema_pool(expressions); } - if constexpr (DATA_TYPE == FileT::CSV) { con1.read_csv(dir_path); } else { con1.read_json(dir_path); } - const auto& original_table = con1.get_table(); // to_fls con1.to_fls(fls_file_path_process_specific); - // decoded rowgroup + // decoded table Connection con2; auto fls_reader = con2.reset().read_fls(fls_file_path_process_specific); auto decoded_table = fls_reader->materialize(); - // Compare rowgroups + auto result = (original_table == *decoded_table); - ASSERT_TRUE(result.is_equal) << "Rowgroups differs. The first not matching column index is: " // - << result.first_failed_column_idx // - << " ❌" // - << "description: " // - << result.description << std::endl; + ASSERT_TRUE(result.is_equal) << "Rowgroups differs. The first not matching column index is: " + << result.first_failed_column_idx << " ❌ description: " << result.description + << std::endl; } void TestConstantness(const vector& constant_indexes) const { @@ -89,49 +86,73 @@ class FastLanesReaderTester : public ::testing::Test { auto first_rowgroup_reader = fls_reader->get_rowgroup_reader(0); const auto& footer = first_rowgroup_reader->get_descriptor(); + const auto* cols = footer.m_column_descriptors(); + const auto ncol = cols ? cols->size() : 0; + for (const auto col_idx : constant_indexes) { - // - auto& col_descriptor = footer.m_column_descriptors[col_idx]; - ASSERT_EQ(col_descriptor->total_size, 0) << col_idx << " should be constant"; + const auto fb_idx = static_cast(col_idx); + ASSERT_LT(fb_idx, ncol) << "column index OOB"; + const auto* col_descriptor = cols->Get(fb_idx); + ASSERT_EQ(col_descriptor->total_size(), 0) << col_idx << " should be constant"; } } void TestEquality(const string_view table) const { TestCorrectness(table); - // decoded rowgroup - Connection con2; - const auto fls_reader = con2.reset().read_fls(fls_file_path_process_specific); - auto first_rowgroup_reader = fls_reader->get_rowgroup_reader(0); - const auto footer = first_rowgroup_reader->get_descriptor(); + Connection con2; + const auto fls_reader = con2.reset().read_fls(fls_file_path_process_specific); + auto first_rowgroup_reader = fls_reader->get_rowgroup_reader(0); + const auto& footer = first_rowgroup_reader->get_descriptor(); + + const auto* cols = footer.m_column_descriptors(); + const auto ncol = cols ? cols->size() : 0; - for (n_t col_idx = 1; col_idx < footer.m_column_descriptors.size(); ++col_idx) { - ASSERT_EQ(footer.m_column_descriptors[col_idx]->total_size, 0) << col_idx << " should be of size 0"; + for (flatbuffers::uoffset_t col_idx = 1; col_idx < ncol; ++col_idx) { + const auto* col_descriptor = cols->Get(col_idx); + ASSERT_EQ(col_descriptor->total_size(), 0) << col_idx << " should be of size 0"; } } void TestEquality(const vector& equal_cols) const { - Connection con; - const auto fls_reader = con.reset().read_fls(fls_file_path_process_specific); - auto first_rowgroup_reader = fls_reader->get_rowgroup_reader(0); - const auto footer = first_rowgroup_reader->get_descriptor(); + Connection con; + const auto fls_reader = con.reset().read_fls(fls_file_path_process_specific); + auto first_rowgroup_reader = fls_reader->get_rowgroup_reader(0); + const auto& footer = first_rowgroup_reader->get_descriptor(); + + const auto* cols = footer.m_column_descriptors(); + const auto ncol = cols ? cols->size() : 0; for (const auto col_index : equal_cols) { - auto& col_descriptor = footer.m_column_descriptors[col_index]; - ASSERT_EQ(col_descriptor->total_size, 0) << "size of column " << col_index << ":" << col_descriptor->name - << " should be 0, as it is equal to another col."; + const auto fb_idx = static_cast(col_index); + ASSERT_LT(fb_idx, ncol) << "column index OOB"; + const auto* col_descriptor = cols->Get(fb_idx); + ASSERT_EQ(col_descriptor->total_size(), 0) + << "size of column " << col_index << " should be 0 (equal to another column)"; } } void TestMap1To1(const vector& target_column_indexes) const { - Connection con; - const auto fls_reader = con.reset().read_fls(fls_file_path_process_specific); - auto first_rowgroup_reader = fls_reader->get_rowgroup_reader(0); - const auto footer = first_rowgroup_reader->get_descriptor(); + Connection con; + const auto fls_reader = con.reset().read_fls(fls_file_path_process_specific); + auto first_rowgroup_reader = fls_reader->get_rowgroup_reader(0); + const auto& footer = first_rowgroup_reader->get_descriptor(); + + const auto* cols = footer.m_column_descriptors(); + const auto ncol = cols ? cols->size() : 0; for (const auto col_index : target_column_indexes) { - auto& col_descriptor = footer.m_column_descriptors[col_index]; - ASSERT_TRUE(is_1_to_1(col_descriptor->encoding_rpn->operator_tokens[0])) << " " << col_index; + const auto fb_idx = static_cast(col_index); + ASSERT_LT(fb_idx, ncol) << "column index OOB"; + const auto* col_descriptor = cols->Get(fb_idx); + + const auto* rpn = col_descriptor->encoding_rpn(); + ASSERT_NE(rpn, nullptr) << "missing encoding_rpn for col " << col_index; + const auto* toks = rpn->operator_tokens(); + ASSERT_NE(toks, nullptr) << "missing operator_tokens for col " << col_index; + ASSERT_GT(toks->size(), 0u) << "no operator tokens for col " << col_index; + + ASSERT_TRUE(is_1_to_1(toks->Get(0))) << " " << col_index; } } @@ -139,7 +160,6 @@ class FastLanesReaderTester : public ::testing::Test { const vector& constant_indexes, const vector& equal_cols, const vector& target_column_indexes) const { - // TestCorrectness(table); TestConstantness(constant_indexes); TestMap1To1(target_column_indexes); @@ -151,6 +171,7 @@ class FastLanesReaderTester : public ::testing::Test { return con.verify_fls(fls_file_path); } }; + } // namespace fastlanes #endif // FLS_TESTER_HPP diff --git a/test/src/fls_reader_tests/rowgroup_size_test.cpp b/test/src/fls_reader_tests/rowgroup_size_test.cpp index 5abbcf9b..f549ea52 100644 --- a/test/src/fls_reader_tests/rowgroup_size_test.cpp +++ b/test/src/fls_reader_tests/rowgroup_size_test.cpp @@ -3,6 +3,7 @@ // ──────────────────────────────────────────────────────── // test/src/fls_reader_tests/rowgroup_size_test.cpp // ──────────────────────────────────────────────────────── +#include "fls/json/fls_json.hpp" #include "fls_tester.hpp" namespace fastlanes { diff --git a/test/src/quick_fuzz_tests/fuzz_config.json b/test/src/quick_fuzz_tests/fuzz_config.json index 40893625..889bc524 100644 --- a/test/src/quick_fuzz_tests/fuzz_config.json +++ b/test/src/quick_fuzz_tests/fuzz_config.json @@ -1,6 +1,6 @@ { "num_cases": 10, - "base_seed": 11, + "base_seed": 12, "delimiter": "|", "min_cols": 1, "max_cols": 2,