From fda6485e190687bc4eb04f42761bafd159523124 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Mon, 21 Sep 2026 03:21:34 -0700 Subject: [PATCH 1/5] Add standalone EloqDoc CMake build and test coverage Build the existing server runtime with Ninja and Data Substrate while preserving the SCons workflow. Retain single-node networking, authentication, sessions, TTL, and local aggregation; omit MongoDB distributed runtimes and unsupported engines. Add native and server-backed validation, repair applicable fixtures and coroutine lifetimes, support GCC 15, retain named test initializers, and match SCons ICU 57.1 collation behavior. Validated the full CMake build, 229 CTests, seven native integration binaries, 884 JavaScript results, server smoke, and focused SCons compatibility checks. Leave the CDC design document and Data Substrate submodule unchanged. --- .github/scripts/common.sh | 5 +- .github/workflows/build.yml | 3 + .github/workflows/ci.yml | 83 +- CMakeLists.txt | 374 +++++++++ cmake/MongoGeneratedSources.cmake | 326 ++++++++ cmake/MongoTests.cmake | 194 +++++ cmake/generate_action_types.py | 33 + cmake/generate_eloqdoc_sources.py | 751 ++++++++++++++++++ cmake/generate_error_codes.py | 161 ++++ cmake/generate_icu_init.py | 29 + cmake/generate_idl.py | 76 ++ cmake/generate_js.py | 25 + cmake/run_legacy_text_generator.py | 33 + cmake/tests/server_smoke.py | 292 +++++++ cmake/tests/test_icu_generator.py | 34 + cmake/tests/test_idl_dependencies.py | 120 +++ cmake/tests/test_scons_test_selection.py | 63 ++ cmake/tests/test_source_graph.py | 180 +++++ docs/how-to-compile.md | 295 ++++++- .../buildscripts/select_eloq_unit_tests.py | 123 +++ scripts/install_dependency_ubuntu2404.sh | 18 +- src/mongo/base/SConscript | 2 + src/mongo/base/error_codes.tpl.h | 6 +- src/mongo/base/error_codes_test.cpp | 54 ++ src/mongo/base/error_codes_test_helper.cpp | 45 ++ src/mongo/bson/simple_bsonobj_comparator.h | 4 +- .../bson/simple_bsonobj_comparator_test.cpp | 11 + src/mongo/db/SConscript | 1 + .../db/auth/authorization_manager_test.cpp | 60 ++ src/mongo/db/catalog/collection_mock.h | 15 + src/mongo/db/catalog/database_holder_mock.h | 2 +- src/mongo/db/catalog/uuid_catalog_test.cpp | 76 +- src/mongo/db/catalog_raii_test.cpp | 12 +- src/mongo/db/commands/conn_pool_stats.cpp | 2 + src/mongo/db/commands/conn_pool_sync.cpp | 2 + src/mongo/db/commands/fsync.cpp | 21 - .../commands/index_filter_commands_test.cpp | 12 +- src/mongo/db/commands/mr.cpp | 6 + .../db/commands/plan_cache_commands_test.cpp | 54 +- .../db/commands/restart_catalog_command.cpp | 6 + ..._feature_compatibility_version_command.cpp | 4 + .../db/concurrency/d_concurrency_test.cpp | 38 + src/mongo/db/coro_sync.cpp | 37 +- src/mongo/db/coro_sync.h | 14 +- src/mongo/db/db.cpp | 39 +- src/mongo/db/ftdc/ftdc_test.cpp | 31 +- ...gical_session_cache_factory_standalone.cpp | 25 + src/mongo/db/logical_session_id_test.cpp | 23 +- .../db/matcher/expression_optimize_test.cpp | 18 +- src/mongo/db/modules/eloq/CMakeLists.txt | 7 + src/mongo/db/mongod_options.cpp | 12 + src/mongo/db/nesting_depth_test.cpp | 48 +- src/mongo/db/op_observer_noop.h | 2 +- src/mongo/db/operation_context.cpp | 4 +- src/mongo/db/operation_context_test.cpp | 96 +++ src/mongo/db/operation_time_tracker.cpp | 16 +- src/mongo/db/operation_time_tracker.h | 3 + src/mongo/db/operation_time_tracker_test.cpp | 20 + src/mongo/db/pipeline/SConscript | 1 + .../db/pipeline/aggregation_context_fixture.h | 2 +- src/mongo/db/pipeline/pipeline.cpp | 21 - src/mongo/db/pipeline/pipeline_sharded.cpp | 60 ++ src/mongo/db/query/canonical_query_test.cpp | 26 +- src/mongo/db/query/collation/SConscript | 25 +- .../collation/collator_factory_icu_test.cpp | 32 +- .../query/collation/collator_interface_icu.h | 5 +- .../collation/collator_interface_icu_test.cpp | 54 +- src/mongo/db/query/get_executor_test.cpp | 12 +- src/mongo/db/query/plan_cache_test.cpp | 96 +-- src/mongo/db/query/query_planner_test.cpp | 10 +- .../db/query/query_planner_test_fixture.cpp | 8 +- .../db/query/query_planner_test_fixture.h | 2 +- src/mongo/db/query/query_request_test.cpp | 44 +- src/mongo/db/record_id.h | 7 +- src/mongo/db/record_id_test.cpp | 12 + .../replication_coordinator_standalone.cpp | 485 +++++++++++ .../repl/replication_coordinator_standalone.h | 253 ++++++ src/mongo/db/repl/replication_info.cpp | 3 + .../db/server_options_server_helpers.cpp | 8 +- src/mongo/db/server_options_test.cpp | 40 + src/mongo/db/service_context.cpp | 11 +- .../db/service_context_d_test_fixture.cpp | 28 +- src/mongo/db/service_context_test_fixture.cpp | 14 +- src/mongo/db/service_context_test_fixture.h | 4 + src/mongo/db/service_entry_point_common.cpp | 4 + src/mongo/db/standalone_runtime_test.cpp | 129 +++ .../ephemeral_for_test_btree_impl.cpp | 48 +- .../ephemeral_for_test_btree_impl.h | 6 + .../ephemeral_for_test_engine.cpp | 72 +- .../ephemeral_for_test_engine.h | 19 + .../ephemeral_for_test_record_store.cpp | 24 +- .../ephemeral_for_test_record_store.h | 9 + .../ephemeral_for_test_record_store_test.cpp | 126 +++ src/mongo/db/storage/key_string_test.cpp | 9 +- src/mongo/db/storage/kv/SConscript | 3 +- .../kv/kv_collection_catalog_entry_test.cpp | 34 +- src/mongo/db/storage/kv/kv_storage_engine.cpp | 10 +- src/mongo/db/storage/storage_engine_init.cpp | 15 +- src/mongo/db/storage/test_harness_helper.h | 6 + src/mongo/db/update/update_driver_test.cpp | 4 +- src/mongo/platform/stack_locator.cpp | 25 +- src/mongo/s/shard_id_test.cpp | 7 +- .../transport/service_executor_adaptive.cpp | 80 +- .../transport/service_executor_adaptive.h | 2 +- src/mongo/transport/service_executor_test.cpp | 151 +++- src/mongo/transport/service_state_machine.cpp | 127 +-- src/mongo/transport/service_state_machine.h | 14 + .../transport/service_state_machine_test.cpp | 128 +++ .../transport/transport_layer_asio_test.cpp | 16 + src/mongo/unittest/death_test.cpp | 10 + src/mongo/util/SConscript | 1 + src/mongo/util/clock_source.cpp | 49 +- src/mongo/util/clock_source.h | 48 ++ .../options_parser/options_parser_test.cpp | 5 +- src/mongo/util/unordered_fast_key_table.h | 2 +- tests/jstests/core/set_param1.js | 7 +- 116 files changed, 5869 insertions(+), 630 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 cmake/MongoGeneratedSources.cmake create mode 100644 cmake/MongoTests.cmake create mode 100644 cmake/generate_action_types.py create mode 100644 cmake/generate_eloqdoc_sources.py create mode 100644 cmake/generate_error_codes.py create mode 100644 cmake/generate_icu_init.py create mode 100644 cmake/generate_idl.py create mode 100644 cmake/generate_js.py create mode 100644 cmake/run_legacy_text_generator.py create mode 100644 cmake/tests/server_smoke.py create mode 100644 cmake/tests/test_icu_generator.py create mode 100644 cmake/tests/test_idl_dependencies.py create mode 100644 cmake/tests/test_scons_test_selection.py create mode 100644 cmake/tests/test_source_graph.py create mode 100644 scripts/buildscripts/select_eloq_unit_tests.py create mode 100644 src/mongo/base/error_codes_test.cpp create mode 100644 src/mongo/base/error_codes_test_helper.cpp create mode 100644 src/mongo/db/logical_session_cache_factory_standalone.cpp create mode 100644 src/mongo/db/pipeline/pipeline_sharded.cpp create mode 100644 src/mongo/db/repl/replication_coordinator_standalone.cpp create mode 100644 src/mongo/db/repl/replication_coordinator_standalone.h create mode 100644 src/mongo/db/standalone_runtime_test.cpp diff --git a/.github/scripts/common.sh b/.github/scripts/common.sh index b21e72453b..3735b0a6d8 100755 --- a/.github/scripts/common.sh +++ b/.github/scripts/common.sh @@ -28,6 +28,7 @@ fi export ELOQ_THIRD_PARTY_REQUIRED="${ELOQ_THIRD_PARTY_REQUIRED:-ON}" export BUILD_JOBS="${BUILD_JOBS:-$(nproc)}" [ "${BUILD_JOBS}" -lt 1 ] && BUILD_JOBS=1 +[ "${BUILD_JOBS}" -gt 8 ] && BUILD_JOBS=8 export CMAKE_BUILD_TIMEOUT_SECONDS="${CMAKE_BUILD_TIMEOUT_SECONDS:-3600}" export SCONS_BUILD_TIMEOUT_SECONDS="${SCONS_BUILD_TIMEOUT_SECONDS:-7200}" export TPCC_TIMEOUT_SECONDS="${TPCC_TIMEOUT_SECONDS:-1200}" @@ -278,7 +279,9 @@ build_eloqdoc() { aarch64|arm64) scons_arch_flags="-march=armv8-a+crc" ;; esac - local scons_third_party_include="-idirafter ${ELOQ_THIRD_PARTY_PREFIX}/include" + # Keep MongoDB's explicit -I paths first, but prefer the selected dependency + # prefix over unrelated copies installed in the compiler's default paths. + local scons_third_party_include="-isystem ${ELOQ_THIRD_PARTY_PREFIX}/include" local scons_cflags="${scons_third_party_include} ${scons_arch_flags} -Wno-nonnull" local scons_cxxflags="${scons_third_party_include} ${scons_arch_flags} -Wno-nonnull -Wno-class-memaccess -Wno-interference-size -Wno-redundant-move" local scons_libpath="${ELOQ_THIRD_PARTY_PREFIX}/lib ${ELOQ_THIRD_PARTY_PREFIX}/lib64" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 87cf0a2fce..41b31d216a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -99,6 +99,9 @@ jobs: VARIANT_ID: ${{ matrix.id }} DATA_STORE_TYPE: ${{ matrix.data_store }} WITH_LOG_STATE: ${{ matrix.log_state }} + # This release job uses the historical SCons packaging path. Pin Python 2 explicitly even + # though the compatibility installer currently enables it by default. + ELOQ_INSTALL_LEGACY_SCONS: 1 steps: - name: Bootstrap checkout tools diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3327223535..49a6e83e8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,86 @@ jobs: arch: ${{ matrix.arch }} cache-scope: ubuntu-dev + cmake-build-smoke: + name: CMake/Ninja single-node tests (amd64) + needs: prime-third-party + runs-on: ubuntu-24.04 + timeout-minutes: 240 + container: + image: eloqdata/ubuntu-dev:24.04 + options: --user root --privileged --security-opt seccomp=unconfined --ulimit memlock=-1:-1 + env: + ELOQ_THIRD_PARTY_PREFIX: /opt/eloq/third_party + CMAKE_PREFIX_PATH: /opt/eloq/third_party + CMAKE_INCLUDE_PATH: /opt/eloq/third_party/include + CMAKE_LIBRARY_PATH: /opt/eloq/third_party/lib + LD_LIBRARY_PATH: /opt/eloq/third_party/lib:/opt/eloq/third_party/lib64 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: false + path: eloqdoc + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Checkout product submodules + run: | + git config --global --add safe.directory '*' + cd "$GITHUB_WORKSPACE/eloqdoc" + bash scripts/checkout_product_submodules.sh + + - name: Setup third-party (cached) + uses: ./eloqdoc/.github/actions/setup-third-party + with: + source-dir: ${{ github.workspace }}/eloqdoc + arch: amd64 + cache-scope: ubuntu-dev + + - name: Limit CMake build and test parallelism to runner capacity + run: | + jobs=$(nproc) + if [ "$jobs" -gt 8 ]; then jobs=8; fi + echo "CMAKE_BUILD_PARALLEL_LEVEL=$jobs" >> "$GITHUB_ENV" + echo "CTEST_PARALLEL_LEVEL=$jobs" >> "$GITHUB_ENV" + + - name: Configure top-level CMake build + working-directory: eloqdoc + run: | + cmake -S . -B build/cmake-ci -G Ninja \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_JOB_POOLS=link_pool=1 \ + -DCMAKE_JOB_POOL_LINK=link_pool \ + -DBUILD_SHARED_LIBS=ON \ + -DELOQDOC_BUILD_TESTS=ON \ + -DELOQ_THIRD_PARTY_PREFIX="$ELOQ_THIRD_PARTY_PREFIX" \ + -DELOQ_THIRD_PARTY_REQUIRED=ON \ + -DWITH_DATA_STORE=ELOQDSS_ROCKSDB \ + -DWITH_LOG_STATE=ROCKSDB \ + -DWITH_LOG_SERVICE=ON \ + -DFORK_HM_PROCESS=ON \ + -DSTATISTICS=ON + + - name: Build server and all retained CMake tests + id: cmake_build + working-directory: eloqdoc + run: | + cmake --build build/cmake-ci \ + --target eloqdoc eloqdoc-tests + + - name: Run all registered CMake tests + working-directory: eloqdoc + run: | + ctest --test-dir build/cmake-ci \ + --output-on-failure + + - name: Exercise the normal server over the MongoDB wire protocol + if: ${{ !cancelled() && steps.cmake_build.outcome == 'success' }} + working-directory: eloqdoc + run: | + python3 -m pip install --target build/cmake-ci/python 'pymongo==4.8.0' + PYTHONPATH="$PWD/build/cmake-ci/python" \ + cmake --build build/cmake-ci --target eloqdoc-server-smoke + ent-ci: needs: prime-third-party concurrency: @@ -136,7 +216,8 @@ jobs: echo "JSTEST_TIMEOUT_SECONDS=7200" >> "$GITHUB_ENV" fi - - name: Build (${{ matrix.build_type }}, ${{ matrix.engine.id }}) + # gh_ci_entry.sh deliberately exercises the historical Python 2/SCons install-core path. + - name: Build with SCons (${{ matrix.build_type }}, ${{ matrix.engine.id }}) run: | bash ./eloqdoc/.github/scripts/gh_ci_entry.sh \ "$S3_ENDPOINT" \ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..2503498923 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,374 @@ +cmake_minimum_required(VERSION 3.24) + +project(EloqDoc VERSION 4.0.3 LANGUAGES C CXX) + +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Build type" FORCE) +endif() + +if(NOT CMAKE_GENERATOR MATCHES "Ninja") + message(FATAL_ERROR "EloqDoc's minimal build supports Ninja generators only. Use: cmake -G Ninja ...") +endif() +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "The minimal EloqDoc build currently supports Linux only") +endif() + +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS ON) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +option(ELOQDOC_BUILD_DATA_SUBSTRATE "Build and link the in-tree Data Substrate" ON) +option(ELOQDOC_BUILD_TESTS + "Build tests for the retained MongoDB runtime and Eloq Mongo module" OFF) +option(ELOQDOC_REGISTER_INTEGRATION_TESTS + "Register tests which require a running EloqDoc server with CTest" OFF) +option(ELOQDOC_ENABLE_ASAN "Build EloqDoc and Data Substrate with AddressSanitizer" OFF) +set(ELOQDOC_VERSION "4.0.3" CACHE STRING "MongoDB compatibility version reported by EloqDoc") +set(ELOQDOC_TEST_CONNECTION_STRING "localhost:27017" CACHE STRING + "MongoDB connection string used by C++ integration tests") +set(WITH_DATA_STORE "ELOQDSS_ELOQSTORE" CACHE STRING "Data Substrate storage backend") +set_property(CACHE WITH_DATA_STORE PROPERTY STRINGS + ROCKSDB ELOQDSS_ROCKSDB ELOQDSS_ROCKSDB_CLOUD_S3 + ELOQDSS_ROCKSDB_CLOUD_GCS ELOQDSS_ELOQSTORE) + +find_package(Python3 3.8 REQUIRED COMPONENTS Interpreter) +find_package(Threads REQUIRED) + +file(GLOB_RECURSE _eloqdoc_sconscript_files CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/src/*SConscript") +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/generate_eloqdoc_sources.py" + ${_eloqdoc_sconscript_files}) + +set(_source_manifest "${CMAKE_CURRENT_BINARY_DIR}/eloqdoc-sources.cmake") +execute_process( + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/generate_eloqdoc_sources.py" + --source-root "${CMAKE_CURRENT_SOURCE_DIR}/src" + --target-arch "${CMAKE_SYSTEM_PROCESSOR}" + --output "${_source_manifest}" + RESULT_VARIABLE _source_manifest_status + ERROR_VARIABLE _source_manifest_error) +if(NOT _source_manifest_status EQUAL 0) + message(FATAL_ERROR "Failed to select the minimal EloqDoc source graph:\n${_source_manifest_error}") +endif() +include("${_source_manifest}") + +list(LENGTH ELOQDOC_MONGO_TARGETS _mongo_target_count) +list(LENGTH ELOQDOC_MONGO_SOURCES _mongo_source_count) +list(LENGTH ELOQDOC_THIRD_PARTY_SOURCES _third_party_source_count) +list(LENGTH ELOQDOC_MONGO_TEST_IDS _mongo_test_count) +message(STATUS + "Minimal EloqDoc graph: ${_mongo_target_count} Mongo modules, " + "${_mongo_source_count} Mongo/Eloq sources, ${_third_party_source_count} vendored sources") +if(ELOQDOC_BUILD_TESTS) + message(STATUS "Single-node MongoDB test graph: ${_mongo_test_count} executables") +endif() + +# Generate the two configure-time headers which SCons used to synthesize. +set(mongo_config_altivec_vec_vbpermq_output_index "// #undef MONGO_CONFIG_ALTIVEC_VEC_VBPERMQ_OUTPUT_INDEX") +set(mongo_config_byte_order "#define MONGO_CONFIG_BYTE_ORDER 1234") +set(mongo_config_debug_build "// #undef MONGO_CONFIG_DEBUG_BUILD") +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(mongo_config_debug_build "#define MONGO_CONFIG_DEBUG_BUILD 1") +endif() +set(mongo_config_has_ssl_set_ecdh_auto "// #undef MONGO_CONFIG_HAS_SSL_SET_ECDH_AUTO") +set(mongo_config_have_execinfo_backtrace "#define MONGO_CONFIG_HAVE_EXECINFO_BACKTRACE 1") +set(mongo_config_have_fips_mode_set "// #undef MONGO_CONFIG_HAVE_FIPS_MODE_SET") +set(mongo_config_have_header_unistd_h "#define MONGO_CONFIG_HAVE_HEADER_UNISTD_H 1") +set(mongo_config_have_memset_s "// #undef MONGO_CONFIG_HAVE_MEMSET_S") +set(mongo_config_have_posix_monotonic_clock "#define MONGO_CONFIG_HAVE_POSIX_MONOTONIC_CLOCK 1") +set(mongo_config_have_pthread_setname_np "#define MONGO_CONFIG_HAVE_PTHREAD_SETNAME_NP 1") +set(mongo_config_have_std_make_unique "#define MONGO_CONFIG_HAVE_STD_MAKE_UNIQUE 1") +set(mongo_config_have_strnlen "#define MONGO_CONFIG_HAVE_STRNLEN 1") +set(mongo_config_max_extended_alignment "#define MONGO_CONFIG_MAX_EXTENDED_ALIGNMENT 64") +set(mongo_config_optimized_build "// #undef MONGO_CONFIG_OPTIMIZED_BUILD") +if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + set(mongo_config_optimized_build "#define MONGO_CONFIG_OPTIMIZED_BUILD 1") +endif() +set(mongo_config_ssl "// #undef MONGO_CONFIG_SSL") +set(mongo_config_ssl_has_asn1_any_definitions "// #undef MONGO_CONFIG_HAVE_ASN1_ANY_DEFINITIONS") +set(mongo_config_ssl_provider "// #undef MONGO_CONFIG_SSL_PROVIDER") +set(mongo_config_wiredtiger_enabled "// #undef MONGO_CONFIG_WIREDTIGER_ENABLED") +file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/generated/mongo/util") +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/config.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/generated/mongo/config.h" @ONLY) + +string(REGEX MATCH "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)$" _version_match "${ELOQDOC_VERSION}") +if(NOT _version_match) + message(FATAL_ERROR "ELOQDOC_VERSION must look like major.minor.patch[-extra]") +endif() +set(mongo_version "${ELOQDOC_VERSION}") +set(mongo_version_major "${CMAKE_MATCH_1}") +set(mongo_version_minor "${CMAKE_MATCH_2}") +set(mongo_version_patch "${CMAKE_MATCH_3}") +set(mongo_version_extra 0) +set(mongo_version_extra_str "${CMAKE_MATCH_4}") +execute_process( + COMMAND git rev-parse HEAD + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE mongo_git_hash + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) +if(NOT mongo_git_hash) + set(mongo_git_hash "unknown") +endif() +set(buildinfo_js_engine "mozjs") +set(buildinfo_allocator "system") +set(buildinfo_modules "{ \"eloq\" }") +set(buildinfo_environment_data "{}") +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/util/version_constants.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/generated/mongo/util/version_constants.h" @ONLY) + +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/MongoGeneratedSources.cmake") + +set(_eloqdoc_server_main "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/dbmain.cpp") +list(REMOVE_ITEM ELOQDOC_COMPILE_SOURCES "${_eloqdoc_server_main}") +set(_eloqdoc_mongo_core_sources) +set(_eloqdoc_eloq_storage_sources) +foreach(_source IN LISTS ELOQDOC_COMPILE_SOURCES) + if(_source MATCHES "/mongo/db/modules/eloq/") + list(APPEND _eloqdoc_eloq_storage_sources "${_source}") + else() + list(APPEND _eloqdoc_mongo_core_sources "${_source}") + endif() +endforeach() + +set(_third_party_absolute_sources) +foreach(_source IN LISTS ELOQDOC_THIRD_PARTY_SOURCES) + list(APPEND _third_party_absolute_sources "${CMAKE_CURRENT_SOURCE_DIR}/src/${_source}") +endforeach() + +add_library(eloqdoc-mongo-third-party OBJECT ${_third_party_absolute_sources}) +add_library(eloqdoc-mongo-runtime OBJECT ${_eloqdoc_mongo_core_sources}) +add_library(eloqdoc-mongo-eloq-storage OBJECT ${_eloqdoc_eloq_storage_sources}) +target_include_directories(eloqdoc-mongo-eloq-storage PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq") + +set(_mongo_include_directories + "${CMAKE_CURRENT_BINARY_DIR}/generated" + "${CMAKE_CURRENT_SOURCE_DIR}/src" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/asio-master/asio/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/boost-1.60.0" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/libstemmer_c/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/libstemmer_c/libstemmer" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/pcre-8.41" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/pcre-8.41/build_posix" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/s2" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/snappy-1.1.3" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/snappy-1.1.3/build_linux" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/timelib-2018.01alpha1" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/tomcrypt-1.18.1/src/headers" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/yaml-cpp-0.5.3/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/zlib-1.2.11") + +# Match src/third_party/icu4c-57.1/source/SConscript for the implementation and +# every consumer of its headers. Collation versions are persisted in index metadata. +set(_icu_root "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/icu4c-57.1/source") +set(_icu_definitions + UCONFIG_NO_BREAK_ITERATION=1 UCONFIG_NO_FORMATTING=1 + UCONFIG_NO_TRANSLITERATION=1 UCONFIG_NO_REGULAR_EXPRESSIONS=1 + U_CHARSET_IS_UTF8=1 U_DISABLE_RENAMING=1 + U_STATIC_IMPLEMENTATION=1 U_USING_ICU_NAMESPACE=0) + +# Match the pinned SpiderMonkey configuration used by SCons. These settings apply only to +# JavaScript implementation files, leaving MongoDB's own headers and compiler options intact. +set(_mozjs_root "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/mozjs-45") +list(APPEND _mongo_include_directories + "${_mozjs_root}/include" + "${_mozjs_root}/platform/${CMAKE_SYSTEM_PROCESSOR}/linux/include") +foreach(_source IN LISTS _third_party_absolute_sources ELOQDOC_COMPILE_SOURCES) + if(_source MATCHES "/third_party/mozjs-45/") + set_property(SOURCE "${_source}" APPEND PROPERTY INCLUDE_DIRECTORIES + "${_mozjs_root}/extract/js/src" "${_mozjs_root}/extract/mfbt" + "${_mozjs_root}/extract/intl/icu/source/common" "${_mozjs_root}/mongo_sources" + "${_mozjs_root}/platform/${CMAKE_SYSTEM_PROCESSOR}/linux/build") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + IMPL_MFBT JS_USE_CUSTOM_ALLOCATOR STATIC_JS_API=1 U_NO_DEFAULT_INCLUDE_UTF_HEADERS=1) + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_OPTIONS + -include js-confdefs.h -Wno-invalid-offsetof) + elseif(_source MATCHES "/mongo/scripting/mozjs/") + set_property(SOURCE "${_source}" APPEND PROPERTY INCLUDE_DIRECTORIES + "${_mozjs_root}/mongo_sources") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + JS_USE_CUSTOM_ALLOCATOR STATIC_JS_API=1) + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_OPTIONS + -include js-config.h -include js/RequiredDefines.h -Wno-invalid-offsetof) + endif() +endforeach() + +function(eloqdoc_configure_mongo_target target) + target_include_directories(${target} PRIVATE ${_mongo_include_directories}) + # ICU's source include directories also contain private names such as util.h. + # Search them after project/substrate headers, but before host system headers. + target_include_directories(${target} SYSTEM PRIVATE + "${_icu_root}/common" "${_icu_root}/i18n") + target_compile_definitions(${target} PRIVATE + _GNU_SOURCE + _XOPEN_SOURCE=700 + BOOST_SYSTEM_NO_DEPRECATED + BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS + ELOQDOC_STANDALONE + ELOQ_MODULE_ENABLED + ELOQ_MODULE_ELOQDOC + EXT_TX_PROC_ENABLED + ${_icu_definitions} + LTC_NO_PROTOTYPES + PCRE_STATIC) + target_compile_options(${target} PRIVATE + $<$:-fno-strict-aliasing> + $<$:-fno-strict-aliasing> + $<$:-fno-omit-frame-pointer> + $<$:-Wno-deprecated-declarations> + $<$:-Wno-nonnull> + $<$:-Wno-class-memaccess> + $<$:-Wno-interference-size> + $<$:-Wno-redundant-move>) +endfunction() + +foreach(_target IN ITEMS + eloqdoc-mongo-third-party + eloqdoc-mongo-runtime + eloqdoc-mongo-eloq-storage) + eloqdoc_configure_mongo_target(${_target}) +endforeach() + +set(_intel_decimal_definitions + DECIMAL_CALL_BY_REFERENCE=0 + DECIMAL_GLOBAL_ROUNDING=0 + DECIMAL_GLOBAL_EXCEPTION_FLAGS=0 + UNCHANGED_BINARY_STATUS_FLAGS=0 + USE_COMPILER_F128_TYPE=0 + USE_COMPILER_F80_TYPE=0 + USE_NATIVE_QUAD_TYPE=0 + LINUX=1 linux=1 gcc=1) +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|aarch64|ppc64le)$") + list(APPEND _intel_decimal_definitions efi2=1 EFI2=1) +endif() +foreach(_source IN LISTS _third_party_absolute_sources) + if(_source MATCHES "/IntelRDFPMathLib20U1/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + ${_intel_decimal_definitions}) + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_OPTIONS -w) + elseif(_source MATCHES "/pcre-8\\.41/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS HAVE_CONFIG_H) + elseif(_source MATCHES "/snappy-1\\.1\\.3/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS HAVE_CONFIG_H) + elseif(_source MATCHES "/timelib-2018\\.01alpha1/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + HAVE_GETTIMEOFDAY HAVE_STRING_H HAVE_DIRENT_H HAVE_SYS_TIME_H HAVE_UNISTD_H) + elseif(_source MATCHES "/zlib-1\\.2\\.11/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + HAVE_STDARG_H HAVE_UNISTD_H) + elseif(_source MATCHES "/s2/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS DEBUG_MODE=false) + elseif(_source MATCHES "/icu4c-57\\.1/source/") + if(_source MATCHES "/common/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + U_COMMON_IMPLEMENTATION=1) + elseif(_source MATCHES "/i18n/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + U_I18N_IMPLEMENTATION=1) + endif() + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_OPTIONS -Wno-sign-compare) + endif() +endforeach() + +add_dependencies(eloqdoc-mongo-runtime eloqdoc-generated-sources) +add_dependencies(eloqdoc-mongo-third-party eloqdoc-generated-sources) +add_dependencies(eloqdoc-mongo-eloq-storage eloqdoc-generated-sources) + +if(ELOQDOC_BUILD_DATA_SUBSTRATE) + set(ELOQ_MODULE_ENABLED ON CACHE BOOL "Register Data Substrate on brpc workers" FORCE) + set(EXT_TX_PROC_ENABLED ON CACHE BOOL "Drive TxProcessors from external workers" FORCE) + set(USE_ASAN "${ELOQDOC_ENABLE_ASAN}" CACHE BOOL "Data Substrate ASan" FORCE) + set(WITH_TESTS OFF CACHE BOOL "Data Substrate tests are maintained independently" FORCE) + set(ENABLE_TESTING OFF CACHE BOOL "Eloq metrics tests are maintained independently" FORCE) + set(WITH_UNIT_TESTS OFF CACHE BOOL "Data-store tests are maintained independently" FORCE) + set(WITH_EXAMPLE OFF CACHE BOOL "EloqStore examples are not part of EloqDoc" FORCE) + set(WITH_DB_STRESS OFF CACHE BOOL "EloqStore stress tools are not part of EloqDoc" FORCE) + add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq" eloq) + # The existing substrate build uses directory-scoped definitions rather than exporting + # all header/ABI switches. Share its actual settings with Mongo consumers without changing + # the independently maintained substrate build or duplicating its backend option mapping. + add_library(eloqdoc-data-substrate INTERFACE) + target_link_libraries(eloqdoc-data-substrate INTERFACE data_substrate) + # Some prebuilt dependencies (notably Abseil) do not carry their own RUNPATH. Use + # transitive RPATH lookup so launching from the build tree needs no LD_LIBRARY_PATH. + target_link_options(eloqdoc-data-substrate INTERFACE "LINKER:--disable-new-dtags") + target_compile_definitions(eloqdoc-data-substrate INTERFACE + $ + $<$:FAULT_INJECTOR>) + set(_data_substrate_target eloqdoc-data-substrate) +else() + add_library(eloqdoc-data-substrate-placeholder INTERFACE) + target_include_directories(eloqdoc-data-substrate-placeholder INTERFACE + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq/data_substrate" + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq/data_substrate/core/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq/data_substrate/tx_service/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq/data_substrate/eloq_metrics/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq/data_substrate/store_handler") + set(_data_substrate_target eloqdoc-data-substrate-placeholder) + message(WARNING + "ELOQDOC_BUILD_DATA_SUBSTRATE=OFF is for source-graph/compiler validation only; " + "the resulting executable cannot link") +endif() + +# MongoDB's Eloq integration includes Data Substrate headers directly. Propagate the selected +# target's public include paths and compile definitions while compiling the Mongo object library, +# not just while linking the final executable. +target_link_libraries(eloqdoc-mongo-runtime PRIVATE ${_data_substrate_target}) +target_link_libraries(eloqdoc-mongo-eloq-storage PRIVATE ${_data_substrate_target}) + +add_executable(eloqdoc + "${_eloqdoc_server_main}" + $ + $ + $) +if(TARGET host_manager) + # Substrate's default startup resolves this helper next to /proc/self/exe. Keep build-tree + # launches working as well as installs, where both executables already go into bin/. + set_target_properties(host_manager PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + target_link_options(host_manager PRIVATE "LINKER:--disable-new-dtags") + add_dependencies(eloqdoc host_manager) +endif() +eloqdoc_configure_mongo_target(eloqdoc) +set_target_properties(eloqdoc PROPERTIES + INSTALL_RPATH "$ORIGIN/../lib" + INSTALL_RPATH_USE_LINK_PATH TRUE) +target_link_libraries(eloqdoc PRIVATE + ${_data_substrate_target} + Threads::Threads + ${CMAKE_DL_LIBS} + m resolv rt) +target_link_options(eloqdoc PRIVATE -rdynamic -Wl,-z,now -Wl,-z,relro -Wl,-z,noexecstack) + +if(ELOQDOC_ENABLE_ASAN) + foreach(_target IN ITEMS + eloqdoc-mongo-third-party + eloqdoc-mongo-runtime + eloqdoc-mongo-eloq-storage + eloqdoc) + target_compile_options(${_target} PRIVATE -fsanitize=address) + target_link_options(${_target} PRIVATE -fsanitize=address) + endforeach() +endif() + +if(ELOQDOC_BUILD_TESTS) + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/MongoTests.cmake") +endif() + +install(TARGETS eloqdoc RUNTIME DESTINATION bin) + +add_custom_target(eloqdoc-source-manifest + COMMAND "${CMAKE_COMMAND}" -E echo + "${_mongo_target_count} selected modules, ${_mongo_source_count} Mongo/Eloq sources, ${_third_party_source_count} vendored sources" + DEPENDS "${_source_manifest}") diff --git a/cmake/MongoGeneratedSources.cmake b/cmake/MongoGeneratedSources.cmake new file mode 100644 index 0000000000..900a497942 --- /dev/null +++ b/cmake/MongoGeneratedSources.cmake @@ -0,0 +1,326 @@ +include_guard(GLOBAL) + +set(ELOQDOC_GENERATED_ROOT "${CMAKE_CURRENT_BINARY_DIR}/generated") +file(MAKE_DIRECTORY + "${ELOQDOC_GENERATED_ROOT}/mongo/base" + "${ELOQDOC_GENERATED_ROOT}/mongo/db/auth" + "${ELOQDOC_GENERATED_ROOT}/mongo/db/fts/unicode" + "${ELOQDOC_GENERATED_ROOT}/mongo/shell") + +option(ELOQDOC_CODEGEN_AT_CONFIGURE + "Generate any missing generated sources during configure, so a freshly \ +configured tree is immediately parseable by clangd and other tooling" ON) + +# Declare a generated source. This always emits the usual build-time rule, so +# editing an .idl/.err/.tpl input still regenerates on the next build. It also +# records the recipe so the configure-time bootstrap at the bottom of this file +# can produce outputs that do not exist yet. +function(_eloqdoc_codegen) + cmake_parse_arguments(ARG "" "MKDIR;DEPFILE;COMMENT" "OUTPUT;COMMAND;DEPENDS" ${ARGN}) + + set(_command_args) + if(ARG_MKDIR) + list(APPEND _command_args COMMAND "${CMAKE_COMMAND}" -E make_directory "${ARG_MKDIR}") + endif() + list(APPEND _command_args COMMAND ${ARG_COMMAND}) + + set(_optional_args) + if(ARG_DEPFILE) + list(APPEND _optional_args DEPFILE "${ARG_DEPFILE}") + endif() + if(ARG_COMMENT) + list(APPEND _optional_args COMMENT "${ARG_COMMENT}") + endif() + + add_custom_command( + OUTPUT ${ARG_OUTPUT} + ${_command_args} + DEPENDS ${ARG_DEPENDS} + ${_optional_args} + VERBATIM) + + get_property(_index GLOBAL PROPERTY ELOQDOC_CODEGEN_COUNT) + if(NOT _index) + set(_index 0) + endif() + set_property(GLOBAL PROPERTY ELOQDOC_CODEGEN_${_index}_OUTPUT "${ARG_OUTPUT}") + set_property(GLOBAL PROPERTY ELOQDOC_CODEGEN_${_index}_COMMAND "${ARG_COMMAND}") + set_property(GLOBAL PROPERTY ELOQDOC_CODEGEN_${_index}_MKDIR "${ARG_MKDIR}") + set_property(GLOBAL PROPERTY ELOQDOC_CODEGEN_${_index}_COMMENT "${ARG_COMMENT}") + math(EXPR _index "${_index} + 1") + set_property(GLOBAL PROPERTY ELOQDOC_CODEGEN_COUNT ${_index}) +endfunction() + +file(GLOB_RECURSE _eloqdoc_idl_compiler_sources CONFIGURE_DEPENDS + "${PROJECT_SOURCE_DIR}/scripts/buildscripts/idl/*.py") + +set(ELOQDOC_GENERATED_OUTPUTS) +set(_eloqdoc_all_source_requests ${ELOQDOC_MONGO_SOURCES}) +if(ELOQDOC_BUILD_TESTS) + list(APPEND _eloqdoc_all_source_requests ${ELOQDOC_MONGO_TEST_SUPPORT_SOURCES}) + foreach(_test_id IN LISTS ELOQDOC_MONGO_TEST_IDS) + list(APPEND _eloqdoc_all_source_requests + ${ELOQDOC_MONGO_TEST_${_test_id}_SOURCES}) + endforeach() +endif() +list(REMOVE_DUPLICATES _eloqdoc_all_source_requests) + +# Embed exactly the same collation data as SCons; do not rely on host ICU data files. +set(_icu_init_cpp "${ELOQDOC_GENERATED_ROOT}/mongo/util/icu_init.cpp") +if(CMAKE_C_BYTE_ORDER STREQUAL "BIG_ENDIAN") + set(_icu_data_name icudt57b.dat) +else() + set(_icu_data_name icudt57l.dat) +endif() +set(_icu_data "${PROJECT_SOURCE_DIR}/src/third_party/icu4c-57.1/source/mongo_sources/${_icu_data_name}") +_eloqdoc_codegen( + OUTPUT "${_icu_init_cpp}" + MKDIR "${ELOQDOC_GENERATED_ROOT}/mongo/util" + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/generate_icu_init.py" + "${PROJECT_SOURCE_DIR}/src/mongo/util/generate_icu_init_cpp.py" + -i "${_icu_data}" -o "${_icu_init_cpp}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/generate_icu_init.py" + "${PROJECT_SOURCE_DIR}/src/mongo/util/generate_icu_init_cpp.py" + "${_icu_data}" + COMMENT "Embedding pinned ICU 57.1 collation data") +list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_icu_init_cpp}") + +foreach(_source IN LISTS _eloqdoc_all_source_requests) + if(_source MATCHES "_gen\\.cpp$") + string(REGEX REPLACE "_gen\\.cpp$" ".idl" _idl_relative "${_source}") + set(_idl_input "${PROJECT_SOURCE_DIR}/src/${_idl_relative}") + if(NOT EXISTS "${_idl_input}") + message(FATAL_ERROR "No IDL input found for generated source ${_source}") + endif() + set(_generated_cpp "${ELOQDOC_GENERATED_ROOT}/${_source}") + string(REGEX REPLACE "\\.cpp$" ".h" _generated_h "${_generated_cpp}") + get_filename_component(_generated_dir "${_generated_cpp}" DIRECTORY) + _eloqdoc_codegen( + OUTPUT "${_generated_cpp}" "${_generated_h}" + MKDIR "${_generated_dir}" + COMMAND "${Python3_EXECUTABLE}" + "${PROJECT_SOURCE_DIR}/cmake/generate_idl.py" + --include "${PROJECT_SOURCE_DIR}/src" + --base_dir "${ELOQDOC_GENERATED_ROOT}" + --target_arch "${CMAKE_SYSTEM_PROCESSOR}" + --header "${_generated_h}" + --output "${_generated_cpp}" + --depfile "${_generated_cpp}.d" + "${_idl_input}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/generate_idl.py" + "${_idl_input}" + ${_eloqdoc_idl_compiler_sources} + DEPFILE "${_generated_cpp}.d" + COMMENT "Generating MongoDB IDL ${_idl_relative}") + list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_generated_cpp}" "${_generated_h}") + endif() +endforeach() + +set(_error_codes_h "${ELOQDOC_GENERATED_ROOT}/mongo/base/error_codes.h") +set(_error_codes_cpp "${ELOQDOC_GENERATED_ROOT}/mongo/base/error_codes.cpp") +_eloqdoc_codegen( + OUTPUT "${_error_codes_h}" "${_error_codes_cpp}" + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/generate_error_codes.py" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.err" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.tpl.h=${_error_codes_h}" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.tpl.cpp=${_error_codes_cpp}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/generate_error_codes.py" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.err" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.tpl.h" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.tpl.cpp" + COMMENT "Generating MongoDB error codes") +list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_error_codes_h}" "${_error_codes_cpp}") + +set(_action_type_h "${ELOQDOC_GENERATED_ROOT}/mongo/db/auth/action_type.h") +set(_action_type_cpp "${ELOQDOC_GENERATED_ROOT}/mongo/db/auth/action_type.cpp") +_eloqdoc_codegen( + OUTPUT "${_action_type_h}" "${_action_type_cpp}" + COMMAND "${Python3_EXECUTABLE}" + "${PROJECT_SOURCE_DIR}/cmake/generate_action_types.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/auth/generate_action_types.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/auth/action_types.txt" + "${_action_type_h}" "${_action_type_cpp}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/generate_action_types.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/auth/generate_action_types.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/auth/action_types.txt" + COMMENT "Generating MongoDB authorization action types") +list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_action_type_h}" "${_action_type_cpp}") + +set(_stop_words_h "${ELOQDOC_GENERATED_ROOT}/mongo/db/fts/stop_words_list.h") +set(_stop_words_cpp "${ELOQDOC_GENERATED_ROOT}/mongo/db/fts/stop_words_list.cpp") +file(GLOB _stop_word_inputs CONFIGURE_DEPENDS + "${PROJECT_SOURCE_DIR}/src/mongo/db/fts/stop_words_*.txt") +_eloqdoc_codegen( + OUTPUT "${_stop_words_h}" "${_stop_words_cpp}" + COMMAND "${Python3_EXECUTABLE}" + "${PROJECT_SOURCE_DIR}/cmake/run_legacy_text_generator.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/fts/generate_stop_words.py" + ${_stop_word_inputs} "${_stop_words_h}" "${_stop_words_cpp}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/run_legacy_text_generator.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/fts/generate_stop_words.py" + ${_stop_word_inputs} + COMMENT "Generating MongoDB text-search stop words") +list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_stop_words_h}" "${_stop_words_cpp}") + +set(_unicode_dir "${ELOQDOC_GENERATED_ROOT}/mongo/db/fts/unicode") +foreach(_kind IN ITEMS casefold delimiter_list diacritic_list) + set(_unicode_output "${_unicode_dir}/codepoints_${_kind}.cpp") + if(_kind STREQUAL "casefold") + set(_unicode_script "gen_casefold_map.py") + set(_unicode_input "${PROJECT_SOURCE_DIR}/src/third_party/unicode-8.0.0/CaseFolding.txt") + elseif(_kind STREQUAL "delimiter_list") + set(_unicode_script "gen_delimiter_list.py") + set(_unicode_input "${PROJECT_SOURCE_DIR}/src/third_party/unicode-8.0.0/PropList.txt") + else() + set(_unicode_script "gen_diacritic_list.py") + set(_unicode_input "${PROJECT_SOURCE_DIR}/src/third_party/unicode-8.0.0/PropList.txt") + endif() + _eloqdoc_codegen( + OUTPUT "${_unicode_output}" + COMMAND "${Python3_EXECUTABLE}" + "${PROJECT_SOURCE_DIR}/cmake/run_legacy_text_generator.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/fts/unicode/${_unicode_script}" + "${_unicode_input}" "${_unicode_output}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/run_legacy_text_generator.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/fts/unicode/${_unicode_script}" + "${PROJECT_SOURCE_DIR}/src/mongo/db/fts/unicode/gen_helper.py" + "${_unicode_input}" + COMMENT "Generating MongoDB Unicode ${_kind} table") + list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_unicode_output}") +endforeach() + +if("mongo/shell/mongo.cpp" IN_LIST ELOQDOC_MONGO_SOURCES) + set(_shell_error_codes "${ELOQDOC_GENERATED_ROOT}/mongo/shell/error_codes.js") + _eloqdoc_codegen( + OUTPUT "${_shell_error_codes}" + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/generate_error_codes.py" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.err" + "${PROJECT_SOURCE_DIR}/src/mongo/shell/error_codes.tpl.js=${_shell_error_codes}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/generate_error_codes.py" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.err" + "${PROJECT_SOURCE_DIR}/src/mongo/shell/error_codes.tpl.js" + COMMENT "Generating shell error-code JavaScript") + + set(_mongo_js "${ELOQDOC_GENERATED_ROOT}/mongo/shell/mongo.cpp") + set(_mongo_js_inputs + assert.js bulk_api.js collection.js crud_api.js db.js explainable.js explain_query.js + mongo.js mr.js query.js session.js types.js utils_auth.js utils.js utils_sh.js) + set(_mongo_js_paths) + foreach(_file IN LISTS _mongo_js_inputs) + list(APPEND _mongo_js_paths "${PROJECT_SOURCE_DIR}/src/mongo/shell/${_file}") + endforeach() + _eloqdoc_codegen( + OUTPUT "${_mongo_js}" + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/generate_js.py" + "${PROJECT_SOURCE_DIR}/scons/site_tools/jstoh.py" + "${_mongo_js}" ${_mongo_js_paths} "${_shell_error_codes}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/generate_js.py" + "${PROJECT_SOURCE_DIR}/scons/site_tools/jstoh.py" + ${_mongo_js_paths} "${_shell_error_codes}" + COMMENT "Embedding MongoDB server JavaScript helpers") + list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_shell_error_codes}" "${_mongo_js}") +endif() + +if("mongo/scripting/mozjs/mongohelpers_js.cpp" IN_LIST ELOQDOC_MONGO_SOURCES) + set(_mongohelpers_js "${ELOQDOC_GENERATED_ROOT}/mongo/scripting/mozjs/mongohelpers_js.cpp") + _eloqdoc_codegen( + OUTPUT "${_mongohelpers_js}" + MKDIR "${ELOQDOC_GENERATED_ROOT}/mongo/scripting/mozjs" + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/generate_js.py" + "${PROJECT_SOURCE_DIR}/scons/site_tools/jstoh.py" + "${_mongohelpers_js}" "${PROJECT_SOURCE_DIR}/src/mongo/scripting/mozjs/mongohelpers.js" + DEPENDS "${PROJECT_SOURCE_DIR}/cmake/generate_js.py" + "${PROJECT_SOURCE_DIR}/scons/site_tools/jstoh.py" + "${PROJECT_SOURCE_DIR}/src/mongo/scripting/mozjs/mongohelpers.js" + COMMENT "Embedding MongoDB mongohelpers JavaScript") + list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_mongohelpers_js}") +endif() + +function(eloqdoc_resolve_mongo_sources output) + set(_resolved) + foreach(_source IN LISTS ARGN) + if(_source MATCHES "_gen\\.cpp$" + OR _source STREQUAL "mongo/base/error_codes.cpp" + OR _source STREQUAL "mongo/db/auth/action_type.cpp" + OR _source STREQUAL "mongo/db/fts/stop_words_list.cpp" + OR _source MATCHES "^mongo/db/fts/unicode/codepoints_(casefold|delimiter_list|diacritic_list)\\.cpp$" + OR _source STREQUAL "mongo/shell/mongo.cpp" + OR _source STREQUAL "mongo/util/icu_init.cpp" + OR _source STREQUAL "mongo/scripting/mozjs/mongohelpers_js.cpp") + list(APPEND _resolved "${ELOQDOC_GENERATED_ROOT}/${_source}") + else() + list(APPEND _resolved "${PROJECT_SOURCE_DIR}/src/${_source}") + endif() + endforeach() + set(${output} "${_resolved}" PARENT_SCOPE) +endfunction() + +eloqdoc_resolve_mongo_sources(ELOQDOC_COMPILE_SOURCES ${ELOQDOC_MONGO_SOURCES}) + +set_source_files_properties(${ELOQDOC_GENERATED_OUTPUTS} PROPERTIES GENERATED TRUE) +add_custom_target(eloqdoc-generated-sources DEPENDS ${ELOQDOC_GENERATED_OUTPUTS}) + +# Configure-time bootstrap. +# +# The rules above only run when something is built. That leaves a freshly +# configured tree with an empty generated/ directory, so every translation unit +# that includes a generated header (mongo/base/error_codes.h, the IDL headers, +# ...) fails to parse. compile_commands.json already points at those headers, so +# clangd reports the whole tree as broken until the first build. +# +# Generate anything that is missing now. Outputs that already exist are left +# alone, so this costs nothing on repeat configures and never competes with the +# build-time rules, which remain responsible for regenerating stale outputs. +if(ELOQDOC_CODEGEN_AT_CONFIGURE) + get_property(_eloqdoc_codegen_count GLOBAL PROPERTY ELOQDOC_CODEGEN_COUNT) + if(NOT _eloqdoc_codegen_count) + set(_eloqdoc_codegen_count 0) + endif() + + set(_eloqdoc_bootstrapped 0) + if(_eloqdoc_codegen_count GREATER 0) + math(EXPR _eloqdoc_codegen_last "${_eloqdoc_codegen_count} - 1") + foreach(_i RANGE ${_eloqdoc_codegen_last}) + get_property(_outputs GLOBAL PROPERTY ELOQDOC_CODEGEN_${_i}_OUTPUT) + set(_needs_generating FALSE) + foreach(_output IN LISTS _outputs) + if(NOT EXISTS "${_output}") + set(_needs_generating TRUE) + break() + endif() + endforeach() + if(NOT _needs_generating) + continue() + endif() + + get_property(_command GLOBAL PROPERTY ELOQDOC_CODEGEN_${_i}_COMMAND) + get_property(_mkdir GLOBAL PROPERTY ELOQDOC_CODEGEN_${_i}_MKDIR) + get_property(_comment GLOBAL PROPERTY ELOQDOC_CODEGEN_${_i}_COMMENT) + if(_mkdir) + file(MAKE_DIRECTORY "${_mkdir}") + endif() + execute_process( + COMMAND ${_command} + RESULT_VARIABLE _codegen_status + ERROR_VARIABLE _codegen_error) + if(NOT _codegen_status EQUAL 0) + message(FATAL_ERROR + "Configure-time code generation failed (${_comment}):\n${_codegen_error}") + endif() + math(EXPR _eloqdoc_bootstrapped "${_eloqdoc_bootstrapped} + 1") + endforeach() + endif() + + if(_eloqdoc_bootstrapped GREATER 0) + message(STATUS + "EloqDoc: generated ${_eloqdoc_bootstrapped} missing source group(s) at configure time") + endif() +endif() diff --git a/cmake/MongoTests.cmake b/cmake/MongoTests.cmake new file mode 100644 index 0000000000..03f82eb9f2 --- /dev/null +++ b/cmake/MongoTests.cmake @@ -0,0 +1,194 @@ +include_guard(GLOBAL) + +enable_testing() +add_test(NAME cmake/standalone-source-graph + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_source_graph.py") +set_tests_properties(cmake/standalone-source-graph PROPERTIES LABELS "cmake;unit") +add_test(NAME cmake/idl-dependencies + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_idl_dependencies.py") +set_tests_properties(cmake/idl-dependencies PROPERTIES LABELS "cmake;unit" TIMEOUT 60) +add_test(NAME cmake/scons-test-selection + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_scons_test_selection.py") +set_tests_properties(cmake/scons-test-selection PROPERTIES LABELS "cmake;unit" TIMEOUT 60) +add_test(NAME cmake/icu-data-generator + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_icu_generator.py") +set_tests_properties(cmake/icu-data-generator PROPERTIES LABELS "cmake;unit" TIMEOUT 60) + +# Opt-in wire-protocol regression fixture; PyMongo is an explicit test-only dependency. +add_custom_target(eloqdoc-server-smoke + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/server_smoke.py" + --server "$" + DEPENDS eloqdoc + USES_TERMINAL) + +# Keep the production objects in archives for tests. Unlike the server executable, a unit test +# should pull only the MongoDB objects it references; this avoids running unrelated global +# initializers and keeps individual test binaries reasonably small. +add_library(eloqdoc-mongo-runtime-test-archive STATIC EXCLUDE_FROM_ALL + $) +add_library(eloqdoc-mongo-third-party-test-archive STATIC EXCLUDE_FROM_ALL + $) +add_library(eloqdoc-mongo-eloq-test-archive STATIC EXCLUDE_FROM_ALL + $) + +# Compile each shared test helper once, but add only the helpers in a test's original SCons +# dependency closure to that executable. This preserves deliberate mock-vs-production choices. +set(_eloqdoc_mongo_test_support_object_targets) +foreach(_relative_source IN LISTS ELOQDOC_MONGO_TEST_SUPPORT_SOURCES) + string(SHA1 _source_id "${_relative_source}") + string(SUBSTRING "${_source_id}" 0 12 _source_id) + set(_support_target "eloqdoc-mongo-test-support-${_source_id}") + eloqdoc_resolve_mongo_sources(_support_source "${_relative_source}") + add_library(${_support_target} OBJECT EXCLUDE_FROM_ALL "${_support_source}") + eloqdoc_configure_mongo_target(${_support_target}) + add_dependencies(${_support_target} eloqdoc-generated-sources) + target_link_libraries(${_support_target} PRIVATE ${_data_substrate_target}) + if(ELOQDOC_ENABLE_ASAN) + target_compile_options(${_support_target} PRIVATE -fsanitize=address) + endif() + list(APPEND _eloqdoc_mongo_test_support_object_targets ${_support_target}) +endforeach() +add_custom_target(eloqdoc-mongo-test-support + DEPENDS ${_eloqdoc_mongo_test_support_object_targets}) + +# These objects implement string-named initializer groups used by every MongoDB test. They must be +# present directly in each executable (rather than discovered from a static archive), and object +# libraries let the complete test suite compile each common source only once. +add_library(eloqdoc-mongo-test-initializers OBJECT EXCLUDE_FROM_ALL + "${PROJECT_SOURCE_DIR}/src/mongo/base/init.cpp" + "${PROJECT_SOURCE_DIR}/src/mongo/base/shim.cpp" + "${PROJECT_SOURCE_DIR}/src/mongo/base/validate_locale.cpp" + "${PROJECT_SOURCE_DIR}/src/mongo/util/options_parser/startup_option_init.cpp") +add_library(eloqdoc-mongo-unit-test-main OBJECT EXCLUDE_FROM_ALL + "${PROJECT_SOURCE_DIR}/src/mongo/unittest/unittest_main.cpp") +add_library(eloqdoc-mongo-integration-test-main OBJECT EXCLUDE_FROM_ALL + "${PROJECT_SOURCE_DIR}/src/mongo/unittest/integration_test_main.cpp") +foreach(_support_target IN ITEMS + eloqdoc-mongo-test-initializers + eloqdoc-mongo-unit-test-main + eloqdoc-mongo-integration-test-main) + eloqdoc_configure_mongo_target(${_support_target}) + add_dependencies(${_support_target} eloqdoc-generated-sources) + target_link_libraries(${_support_target} PRIVATE ${_data_substrate_target}) + if(ELOQDOC_ENABLE_ASAN) + target_compile_options(${_support_target} PRIVATE -fsanitize=address) + endif() +endforeach() + +set(_eloqdoc_mongo_test_targets) +set(_eloqdoc_mongo_test_object_targets) +set(_eloqdoc_mongo_test_object_targets_without_eloq) +set(_eloqdoc_mongo_unit_test_count 0) +set(_eloqdoc_mongo_integration_test_count 0) + +foreach(_test_id IN LISTS ELOQDOC_MONGO_TEST_IDS) + set(_name "${ELOQDOC_MONGO_TEST_${_test_id}_NAME}") + set(_kind "${ELOQDOC_MONGO_TEST_${_test_id}_KIND}") + set(_requires_eloq "${ELOQDOC_MONGO_TEST_${_test_id}_REQUIRES_ELOQ}") + set(_relative_sources ${ELOQDOC_MONGO_TEST_${_test_id}_SOURCES}) + set(_relative_support_sources ${ELOQDOC_MONGO_TEST_${_test_id}_SUPPORT_SOURCES}) + eloqdoc_resolve_mongo_sources(_test_sources ${_relative_sources}) + set(_support_objects) + foreach(_relative_support_source IN LISTS _relative_support_sources) + string(SHA1 _support_source_id "${_relative_support_source}") + string(SUBSTRING "${_support_source_id}" 0 12 _support_source_id) + list(APPEND _support_objects + $) + endforeach() + + string(REPLACE "/" "-" _target_suffix "${_name}") + string(REPLACE "_" "-" _target_suffix "${_target_suffix}") + set(_target "eloqdoc-test-${_target_suffix}") + + if(_kind STREQUAL "unit") + set(_main_objects + $) + math(EXPR _eloqdoc_mongo_unit_test_count "${_eloqdoc_mongo_unit_test_count} + 1") + else() + set(_main_objects + $) + math(EXPR _eloqdoc_mongo_integration_test_count + "${_eloqdoc_mongo_integration_test_count} + 1") + endif() + set(_object_target "${_target}-objects") + add_library(${_object_target} OBJECT EXCLUDE_FROM_ALL ${_test_sources}) + eloqdoc_configure_mongo_target(${_object_target}) + add_dependencies(${_object_target} eloqdoc-generated-sources) + target_link_libraries(${_object_target} PRIVATE ${_data_substrate_target}) + if(ELOQDOC_ENABLE_ASAN) + target_compile_options(${_object_target} PRIVATE -fsanitize=address) + endif() + + add_executable(${_target} EXCLUDE_FROM_ALL + $ + ${_support_objects} + ${_main_objects} + $) + set_target_properties(${_target} PROPERTIES + OUTPUT_NAME "${_target_suffix}" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tests/mongo") + eloqdoc_configure_mongo_target(${_target}) + add_dependencies(${_target} eloqdoc-generated-sources) + set(_test_archives + eloqdoc-mongo-runtime-test-archive + eloqdoc-mongo-third-party-test-archive) + if(_requires_eloq) + list(APPEND _test_archives eloqdoc-mongo-eloq-test-archive) + endif() + string(JOIN "," _test_archive_group ${_test_archives}) + target_link_libraries(${_target} PRIVATE + "$" + ${_data_substrate_target} + Threads::Threads + ${CMAKE_DL_LIBS} + m resolv rt) + target_link_options(${_target} PRIVATE + -rdynamic -Wl,-z,now -Wl,-z,relro -Wl,-z,noexecstack) + if(ELOQDOC_ENABLE_ASAN) + target_compile_options(${_target} PRIVATE -fsanitize=address) + target_link_options(${_target} PRIVATE -fsanitize=address) + endif() + + if(_kind STREQUAL "unit") + add_test(NAME "${_name}" COMMAND ${_target}) + set_tests_properties("${_name}" PROPERTIES LABELS "mongo;unit" TIMEOUT 300 + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}") + if(_requires_eloq) + set_property(TEST "${_name}" APPEND PROPERTY LABELS "eloq") + endif() + else() + # Exercise integration-only initializer retention even without a running server. + add_test(NAME "${_name}/startup-options" COMMAND ${_target} --help) + set_tests_properties("${_name}/startup-options" PROPERTIES + LABELS "mongo;integration-startup" TIMEOUT 30 + PASS_REGULAR_EXPRESSION "--connectionString") + if(ELOQDOC_REGISTER_INTEGRATION_TESTS) + add_test(NAME "${_name}" COMMAND ${_target} + "--connectionString=${ELOQDOC_TEST_CONNECTION_STRING}") + set_tests_properties("${_name}" PROPERTIES + LABELS "mongo;integration" TIMEOUT 600) + endif() + endif() + + list(APPEND _eloqdoc_mongo_test_targets ${_target}) + list(APPEND _eloqdoc_mongo_test_object_targets ${_object_target}) + if(NOT _requires_eloq) + list(APPEND _eloqdoc_mongo_test_object_targets_without_eloq ${_object_target}) + endif() +endforeach() + +add_custom_target(eloqdoc-mongo-test-objects + DEPENDS ${_eloqdoc_mongo_test_object_targets}) +add_custom_target(eloqdoc-mongo-test-objects-without-eloq + DEPENDS ${_eloqdoc_mongo_test_object_targets_without_eloq}) +add_custom_target(eloqdoc-mongo-tests DEPENDS ${_eloqdoc_mongo_test_targets}) +add_custom_target(eloqdoc-tests DEPENDS ${_eloqdoc_mongo_test_targets}) + +message(STATUS + "MongoDB tests: ${_eloqdoc_mongo_unit_test_count} unit, " + "${_eloqdoc_mongo_integration_test_count} integration") +if(NOT ELOQDOC_REGISTER_INTEGRATION_TESTS AND _eloqdoc_mongo_integration_test_count GREATER 0) + message(STATUS + "MongoDB integration binaries will be built but not registered with CTest; " + "set ELOQDOC_REGISTER_INTEGRATION_TESTS=ON when a server fixture is available") +endif() diff --git a/cmake/generate_action_types.py b/cmake/generate_action_types.py new file mode 100644 index 0000000000..2c5a843009 --- /dev/null +++ b/cmake/generate_action_types.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Run MongoDB's Python 2 action-type generator for the CMake build.""" + +import sys +from pathlib import Path + + +def main(): + if len(sys.argv) < 2: + raise SystemExit("usage: generate_action_types.py SCRIPT [ARG ...]") + + script = Path(sys.argv[1]) + source = script.read_text(encoding="utf-8") + replacements = { + " print 'Duplicate actionType %s\\n' % actionType": + " print('Duplicate actionType %s\\n' % actionType)", + ' print "Usage: generate_action_types.py ' + '
"': + ' print("Usage: generate_action_types.py ' + '
")', + } + for old, new in replacements.items(): + if source.count(old) != 1: + raise RuntimeError("unexpected action-type generator format") + source = source.replace(old, new) + + sys.argv = sys.argv[1:] + globals_ = {"__file__": str(script), "__name__": "__main__"} + exec(compile(source, str(script), "exec"), globals_) + + +if __name__ == "__main__": + main() diff --git a/cmake/generate_eloqdoc_sources.py b/cmake/generate_eloqdoc_sources.py new file mode 100644 index 0000000000..abb39f29d9 --- /dev/null +++ b/cmake/generate_eloqdoc_sources.py @@ -0,0 +1,751 @@ +#!/usr/bin/env python3 +"""Extract the minimal EloqDoc source closure from MongoDB 4.0 SConscript files. + +The build itself is CMake/Ninja-only. This small configure-time translator keeps +the source manifest tied to MongoDB's fine-grained library declarations while +selecting the normal standalone server, the Eloq storage engine, and their +transitive dependencies. It never imports or executes SCons. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import os +import platform +from pathlib import Path, PurePosixPath +from typing import Any, Dict, Iterable, List, MutableMapping, Set + + +Target = Dict[str, Any] + +FORBIDDEN_TARGET_PREFIXES = ( + "mongo/mongos", + "mongo/db/repl/bgsync", + "mongo/db/repl/initial_syncer", + "mongo/db/repl/repl_coordinator_impl", + "mongo/db/repl/serveronly_repl", + "mongo/db/storage/mobile/storage_mobile", + "mongo/db/storage/mmap_v1/storage_mmapv1", + "mongo/db/storage/wiredtiger/", + "third_party/wiredtiger/", + "mongo/embedded/embedded", + "mongo/embedded/mongoed", + "mongo/db/s/balancer", + "mongo/db/s/commands_db_s", + "mongo/db/s/sharding_catalog_manager", + "mongo/s/commands/", + "mongo/s/sharding_legacy_api", +) + +EXCLUDED_TEST_TARGET_PREFIXES = ( + "mongo/db/repl/", + "mongo/db/s/", + "mongo/db/storage/mmap_v1/", + "mongo/db/storage/mobile/", + "mongo/db/storage/wiredtiger/", + "mongo/s/", + "mongo/shell/", +) + +# Distributed-only suites outside the repl/ and sharding directories. Their libraries +# may still supply shared types to the standalone server; that does not make these +# feature-specific tests part of EloqDoc validation. SCons targets remain unchanged. +EXCLUDED_DISTRIBUTED_TEST_TARGETS = { + "mongo/client/dbclient_rs_test": "replica-set client routing/failover", + "mongo/client/replica_set_monitor_test": "replica-set topology monitoring", + "mongo/db/keys_collection_document_test": "cluster-time signing keys", + "mongo/db/operation_time_tracker_test": "operation times from remote MongoDB servers", + "mongo/db/time_proof_service_test": "cluster-time signing/verification", + "mongo/rpc/config_server_metadata_test": "config-server metadata", + "mongo/rpc/oplog_query_metadata_test": "replication oplog-query metadata", + "mongo/rpc/repl_set_metadata_test": "replica-set metadata", +} + +SOURCE_REPLACEMENTS = { + "mongo/db/ftdc/ftdc_system_stats_${TARGET_OS}.cpp": "mongo/db/ftdc/ftdc_system_stats_linux.cpp", + "mongo/db/storage/mmap_v1/mmap_${TARGET_OS_FAMILY}.cpp": "mongo/db/storage/mmap_v1/mmap_posix.cpp", + "mongo/crypto/sha_block_${MONGO_CRYPTO}.cpp": "mongo/crypto/sha_block_tom.cpp", + "mongo/db/storage/storage_engine_lock_file_${TARGET_OS_FAMILY}.cpp": + "mongo/db/storage/storage_engine_lock_file_posix.cpp", + "mongo/platform/shared_library_${TARGET_OS_FAMILY}.cpp": + "mongo/platform/shared_library_posix.cpp", + "mongo/platform/stack_locator_${TARGET_OS}.cpp": "mongo/platform/stack_locator_linux.cpp", + "mongo/util/processinfo_${TARGET_OS}.cpp": "mongo/util/processinfo_linux.cpp", + "mongo/util/stacktrace_${TARGET_OS_FAMILY}.cpp": "mongo/util/stacktrace_posix.cpp", +} + +GENERATED_WITHOUT_SOURCE_FILE = { + "mongo/base/error_codes.cpp", + "mongo/db/auth/action_type.cpp", + "mongo/db/fts/stop_words_list.cpp", + "mongo/db/fts/unicode/codepoints_casefold.cpp", + "mongo/db/fts/unicode/codepoints_delimiter_list.cpp", + "mongo/db/fts/unicode/codepoints_diacritic_list.cpp", + "mongo/shell/mongo.cpp", + "mongo/scripting/mozjs/mongohelpers_js.cpp", + "mongo/util/icu_init.cpp", +} + + +def _flatten(value: Any) -> List[str]: + if isinstance(value, list): + result: List[str] = [] + for item in value: + result.extend(_flatten(item)) + return result + return [value] if isinstance(value, str) else [] + + +class StaticEvaluator: + """Evaluate the literal subset used for SConscript source/dependency lists.""" + + def __init__(self, variables: MutableMapping[str, Any]): + self.variables = variables + + def eval(self, node: ast.AST | None, locals_: Dict[str, Any] | None = None) -> Any: + if node is None: + return [] + local_values = locals_ or {} + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, (ast.List, ast.Tuple)): + return [self.eval(item, local_values) for item in node.elts] + if isinstance(node, ast.Name): + return local_values.get(node.id, self.variables.get(node.id, [])) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not bool(self.eval(node.operand, local_values)) + if isinstance(node, ast.BinOp): + left = self.eval(node.left, local_values) + right = self.eval(node.right, local_values) + if isinstance(node.op, ast.Add): + if isinstance(left, list) or isinstance(right, list): + return _flatten(left) + _flatten(right) + return left + right + if isinstance(node.op, ast.Mod): + try: + return left % right + except (TypeError, ValueError): + return [] + if isinstance(node, ast.IfExp): + branch = node.body if self.eval(node.test, local_values) else node.orelse + return self.eval(branch, local_values) + if isinstance(node, ast.Compare) and len(node.ops) == 1: + left = self.eval(node.left, local_values) + right = self.eval(node.comparators[0], local_values) + if isinstance(node.ops[0], ast.Eq): + return left == right + if isinstance(node.ops[0], ast.NotEq): + return left != right + if isinstance(node.ops[0], ast.In): + return left in right + if isinstance(node.ops[0], ast.NotIn): + return left not in right + if isinstance(node, ast.BoolOp): + values = [bool(self.eval(item, local_values)) for item in node.values] + return all(values) if isinstance(node.op, ast.And) else any(values) + if isinstance(node, ast.Subscript): + value = self.eval(node.value, local_values) + index = self.eval(node.slice, local_values) + try: + return value[index] + except (IndexError, KeyError, TypeError): + return [] + if isinstance(node, ast.ListComp) and len(node.generators) == 1: + generator = node.generators[0] + if not isinstance(generator.target, ast.Name): + return [] + result = [] + for item in _flatten(self.eval(generator.iter, local_values)): + nested = dict(local_values) + nested[generator.target.id] = item + result.append(self.eval(node.elt, nested)) + return result + if isinstance(node, ast.Call): + if isinstance(node.func, ast.Name) and node.func.id == "use_system_version_of_library": + # Match SCons' pinned dependencies, including ICU: its version is stored in + # index collation specifications, so using the host's ICU changes compatibility. + return False + if isinstance(node.func, ast.Attribute): + method = node.func.attr + if method == "TargetOSIs": + return any(self.eval(arg, local_values) in ("linux", "posix") for arg in node.args) + if method == "ToolchainIs": + return any(self.eval(arg, local_values) in ("GCC", "gcc") for arg in node.args) + if method == "Idlc": + source = Path(str(self.eval(node.args[0], local_values))) + generated_base = source.with_suffix("").as_posix() + "_gen" + # SCons returns [generated_cpp, generated_header]; Mongo's declarations select + # element zero. Preserve that shape so the surrounding AST subscript works. + return [generated_base + ".cpp", generated_base + ".h"] + if method == "WindowsResourceFile": + return [] + return [] + + +def _active_statements( + statements: Iterable[ast.stmt], evaluator: StaticEvaluator +) -> Iterable[ast.stmt]: + """Yield statements from branches which are active in the Linux build profile.""" + for statement in statements: + if isinstance(statement, ast.If): + branch = statement.body if evaluator.eval(statement.test) else statement.orelse + yield from _active_statements(branch, evaluator) + else: + yield statement + + +def _collect_variables(tree: ast.Module, target_arch: str) -> Dict[str, Any]: + variables: Dict[str, Any] = { + "debugBuild": False, + "env": { + "MONGO_ALLOCATOR": "system", + "MONGO_BUILD_SASL_CLIENT": True, + "MONGO_CRYPTO": "tom", + "MONGO_HAVE_LIBMONGOC": False, + "MONGO_MODULES": ["eloq"], + "MODULE_BANNERS": [], + "TARGET_ARCH": target_arch, + "TARGET_OS": "linux", + }, + "free_monitoring": "off", + "mmapv1": False, + "serverJs": True, + "usemozjs": True, + "wiredtiger": False, + } + evaluator = StaticEvaluator(variables) + # All source variables in this MongoDB branch are literal lists declared before use. Repeating + # the walk resolves lists which refer to an earlier list. Walking only the selected branch is + # important: several scripts assign Linux and Windows dependencies to the same variable. + for _ in range(3): + for statement in _active_statements(tree.body, evaluator): + for node in ast.walk(statement): + if isinstance(node, ast.If): + # Nested conditionals are handled by _active_statements, not ast.walk. + continue + if not isinstance(node, ast.Assign) or len(node.targets) != 1: + continue + target = node.targets[0] + if isinstance(target, ast.Name): + value = evaluator.eval(node.value) + if value != []: + variables[target.id] = value + return variables + + +def _canonical_dependency(base: str, dependency: str) -> str: + if dependency.startswith("$BUILD_DIR/"): + return dependency[len("$BUILD_DIR/") :] + if dependency.startswith("#/"): + return dependency[2:] + return os.path.normpath((PurePosixPath(base) / dependency).as_posix()) + + +def _canonical_source(base: str, source: str) -> str: + if source.startswith("$BUILD_DIR/"): + return source[len("$BUILD_DIR/") :] + return (PurePosixPath(base) / source).as_posix() + + +def load_targets(source_root: Path, target_arch: str = platform.machine()) -> Dict[str, Target]: + targets: Dict[str, Target] = {} + for script in source_root.rglob("SConscript"): + tree = ast.parse(script.read_text(encoding="utf-8"), filename=str(script)) + evaluator = StaticEvaluator(_collect_variables(tree, target_arch)) + base = script.parent.relative_to(source_root).as_posix() + for statement in _active_statements(tree.body, evaluator): + for node in ast.walk(statement): + if isinstance(node, ast.If): + # Nested conditionals are handled by _active_statements, not ast.walk. + continue + if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute): + continue + kind = node.func.attr + if kind not in ("Library", "Program", "CppUnitTest", "CppIntegrationTest"): + continue + keywords = {item.arg: item.value for item in node.keywords} + target_node = keywords.get("target", node.args[0] if node.args else None) + source_node = keywords.get("source", node.args[1] if len(node.args) > 1 else None) + names = _flatten(evaluator.eval(target_node)) + sources = _flatten(evaluator.eval(source_node)) + dependencies: List[str] = [] + for field in ("LIBDEPS", "LIBDEPS_PRIVATE", "LIBDEPS_INTERFACE"): + dependencies.extend(_flatten(evaluator.eval(keywords.get(field)))) + if kind == "CppUnitTest": + dependencies.append("$BUILD_DIR/mongo/unittest/unittest_main") + elif kind == "CppIntegrationTest": + dependencies.append("$BUILD_DIR/mongo/unittest/integration_test_main") + for name in names: + canonical_name = (PurePosixPath(base) / name).as_posix() + record = targets.setdefault( + canonical_name, {"sources": [], "dependencies": [], "kind": kind} + ) + record["sources"].extend(_canonical_source(base, item) for item in sources) + record["dependencies"].extend( + _canonical_dependency(base, item) for item in dependencies + ) + return targets + + +def select_minimal_graph(targets: Dict[str, Target], source_root: Path | None = None, + target_arch: str = platform.machine()) -> Set[str]: + # Keep db.cpp, ServiceEntryPointMongod, authentication, maintenance and JavaScript. + # Only the replication/sharding boundaries and alternative engines are removed. + distributed = { + "mongo/db/repl/bgsync", "mongo/db/repl/oplog_buffer_blocking_queue", + "mongo/db/repl/oplog_buffer_collection", "mongo/db/repl/oplog_buffer_proxy", + "mongo/db/repl/repl_coordinator_impl", "mongo/db/repl/rs_rollback", + "mongo/db/repl/rslog", "mongo/db/repl/serveronly_repl", + "mongo/db/repl/oplog_application", "mongo/db/repl/topology_coordinator", + "mongo/db/repl/repl_set_commands", "mongo/db/s/commands_db_s", + "mongo/db/s/balancer", "mongo/db/s/op_observer_sharding_impl", + "mongo/db/s/sharding_catalog_manager", "mongo/s/sharding_legacy_api", + "mongo/s/catalog/sharding_catalog_client_impl", + "mongo/s/commands/shared_cluster_commands", + "mongo/db/storage/mmap_v1/storage_mmapv1", + "mongo/db/storage/devnull/storage_devnull", + "mongo/db/storage/ephemeral_for_test/storage_ephemeral_for_test", + } + for name in ("mongo/mongodmain", "mongo/db/serveronly", "mongo/db/commands/mongod", + "mongo/db/commands/mongod_fsync", "mongo/db/commands/servers"): + targets[name]["dependencies"] = [ + dep for dep in targets[name]["dependencies"] if dep not in distributed + ] + targets["mongo/mongodmain"]["dependencies"].extend([ + "mongo/db/modules/eloq/storage_eloq", "mongo/db/repl/repl_coordinator_standalone", + "mongo/db/op_observer_impl", "mongo/db/repl/replication_info", + ]) + # This target also registers the standalone isMaster handshake and diagnostics. It does + # not use OplogReader; the old dependency must not retain a replica-set oplog client. + targets["mongo/db/repl/replication_info"]["dependencies"] = [ + dep for dep in targets["mongo/db/repl/replication_info"]["dependencies"] + if dep != "mongo/db/repl/oplogreader" + ] + # The interface supplies shared write-concern constants without the catalog client runtime. + targets["mongo/db/keys_collection_client_direct"]["dependencies"].append( + "mongo/s/catalog/sharding_catalog_client") + targets["mongo/db/repl/repl_coordinator_standalone"] = { + "sources": ["mongo/db/repl/replication_coordinator_standalone.cpp"], + "dependencies": ["mongo/db/repl/repl_coordinator_interface", + "mongo/db/repl/repl_settings", "mongo/db/global_settings"], + "kind": "Library", + } + targets["mongo/db/standalone_runtime_test"] = { + "sources": ["mongo/db/standalone_runtime_test.cpp"], + "dependencies": ["mongo/db/repl/repl_coordinator_standalone", + "mongo/unittest/unittest_main"], + "kind": "CppUnitTest", + } + for name in ("mongo/mongodmain", "mongo/db/serveronly"): + targets[name]["dependencies"] = [ + "mongo/db/s/sharding_runtime_d_embedded" + if dep == "mongo/db/s/sharding_runtime_d" else dep + for dep in targets[name]["dependencies"] + ] + # Replica-set-only commands must not register on this standalone server. + targets["mongo/db/commands/mongod"]["sources"] = [ + src for src in targets["mongo/db/commands/mongod"]["sources"] + if Path(src).name not in {"resize_oplog.cpp", "oplog_note.cpp", "dbcheck.cpp"} + ] + targets["mongo/db/commands/mongod"]["dependencies"] = [ + dep for dep in targets["mongo/db/commands/mongod"]["dependencies"] + if dep != "mongo/db/repl/dbcheck" + ] + # Substitute the small factory at the link boundary instead of branching shared code. + targets["mongo/db/logical_session_cache_factory_mongod"]["sources"] = [ + "mongo/db/logical_session_cache_factory_standalone.cpp", + ] + targets["mongo/db/logical_session_cache_factory_mongod"]["dependencies"] = [ + "mongo/db/logical_session_cache_impl", "mongo/db/service_liaison_mongod", + "mongo/db/sessions_collection_standalone", + ] + + # The common request path needs CollectionShardingState, not the shard-server runtime. + # Reuse MongoDB's unsharded metadata implementation without its embedded application. + entry_point = targets["mongo/db/service_entry_point_common"] + entry_point["dependencies"] = [ + ( + "mongo/db/s/sharding_runtime_d_embedded" + if item == "mongo/db/s/sharding_runtime_d" + else item + ) + for item in entry_point["dependencies"] + ] + + # Keep local Pipeline functionality, but omit the separately compiled sharding entry points, + # distributed planner, and remote cursor implementation. No rejection stubs are needed: + # pipeline.cpp no longer references the distributed planner. + pipeline = targets["mongo/db/pipeline/pipeline"] + pipeline["sources"] = [ + item for item in pipeline["sources"] + if item + not in { + "mongo/db/pipeline/pipeline_sharded.cpp", + "mongo/db/pipeline/cluster_aggregation_planner.cpp", + "mongo/db/pipeline/document_source_internal_split_pipeline.cpp", + "mongo/db/pipeline/document_source_merge_cursors.cpp", + } + ] + # pipeline.h exposes AsyncResultsMergerParams even in the local-only planner. Retain the IDL + # value type (but not the async-results-merger runtime) so clean builds generate its header and + # provide any serialization symbols used by the shared pipeline API. + pipeline["sources"].append("mongo/s/query/async_results_merger_params_gen.cpp") + pipeline["dependencies"] = [ + item + for item in pipeline["dependencies"] + if item != "mongo/s/query/async_results_merger" + ] + + # dbcommands.cpp consumes the generated profile request type. Full mongod reaches it through + # the mongod command bundle, while the embedded command bundle omitted that edge. + targets["mongo/db/commands/standalone"]["dependencies"].append( + "mongo/db/commands/profile_common" + ) + + # catalog_impl has no Balancer symbol references. This legacy LIBDEPS edge exists to retain + # static initializers in full mongod; the single-node executable must not retain them. + catalog_impl = targets["mongo/db/catalog/catalog_impl"] + catalog_impl["dependencies"] = [ + item for item in catalog_impl["dependencies"] if item != "mongo/db/s/balancer" + ] + + # Standalone startup does not initialize a sharded logical-time key client. + keys_manager = targets["mongo/db/keys_collection_manager"] + keys_manager["dependencies"] = [ + item + for item in keys_manager["dependencies"] + if item != "mongo/db/keys_collection_client_sharded" + ] + + # The SpiderMonkey SConscript appends generated unified translation units with Glob(). + # Keep that explicit platform source set without executing SCons. + mozjs = targets["third_party/mozjs-45/mozjs"] + if target_arch == "x86_64": + mozjs["sources"].append( + "third_party/mozjs-45/extract/js/src/jit/x86-shared/Disassembler-x86-shared.cpp") + mozjs_root = source_root or Path(__file__).resolve().parents[1] / "src" + mozjs_platform = mozjs_root / "third_party/mozjs-45/platform" / target_arch / "linux/build" + if not mozjs_platform.is_dir(): + raise RuntimeError(f"No pinned SpiderMonkey configuration for {target_arch}/linux") + mozjs["sources"].extend( + path.relative_to(mozjs_root).as_posix() + for path in sorted(mozjs_platform.glob("*.cpp")) + ) + + # TLS is excluded from this single-node profile, so use MongoDB's small vendored TomCrypt + # implementation for SCRAM hashes rather than compiling an SSL provider. + targets["mongo/crypto/sha_block_${MONGO_CRYPTO}"] = { + "sources": ["mongo/crypto/sha_block_tom.cpp"], + "dependencies": [ + "mongo/base", + "mongo/crypto/sha1_block", + "mongo/crypto/sha256_block", + "third_party/shim_tomcrypt", + ], + "kind": "Library", + } + + # SCons carries interface dependencies on cloned environments. Recreate the subset used by + # this build explicitly; these are ordinary vendored libraries, not SCons runtime behavior. + interface_dependencies = { + "third_party/shim_asio": ["third_party/asio-master/asio"], + "third_party/shim_boost": [ + "third_party/boost-1.60.0/boost_program_options", + "third_party/boost-1.60.0/boost_filesystem", + "third_party/boost-1.60.0/boost_system", + "third_party/boost-1.60.0/boost_iostreams", + ], + "third_party/shim_intel_decimal128": [ + "third_party/IntelRDFPMathLib20U1/intel_decimal128" + ], + "third_party/shim_icu": ["third_party/icu4c-57.1/source/icu_i18n"], + "third_party/shim_pcrecpp": ["third_party/pcre-8.41/pcrecpp"], + "third_party/shim_mozjs": ["third_party/mozjs-45/mozjs"], + "third_party/shim_snappy": ["third_party/snappy-1.1.3/snappy"], + "third_party/shim_stemmer": ["third_party/libstemmer_c/stemmer"], + "third_party/shim_timelib": ["third_party/timelib-2018.01alpha1/timelib"], + "third_party/shim_tomcrypt": ["third_party/tomcrypt-1.18.1/tomcrypt"], + "third_party/shim_yaml": ["third_party/yaml-cpp-0.5.3/yaml"], + "third_party/shim_zlib": ["third_party/zlib-1.2.11/zlib"], + } + for target, dependencies in interface_dependencies.items(): + targets[target]["dependencies"].extend(dependencies) + + selected: Set[str] = set() + pending = ["mongo/eloqdoc"] + while pending: + name = pending.pop() + if name in selected: + continue + if name not in targets: + raise RuntimeError(f"Minimal EloqDoc graph references unknown target: {name}") + selected.add(name) + pending.extend(targets[name]["dependencies"]) + + bad = sorted( + name + for name in selected + if any(name.startswith(item) for item in FORBIDDEN_TARGET_PREFIXES) + ) + if bad: + raise RuntimeError("Forbidden distributed/alternate-engine targets selected: " + ", ".join(bad)) + return selected + + +def _dependency_closure(targets: Dict[str, Target], roots: Iterable[str]) -> Set[str] | None: + selected: Set[str] = set() + pending = list(roots) + while pending: + name = pending.pop() + if name in selected: + continue + if name not in targets: + return None + selected.add(name) + pending.extend(targets[name]["dependencies"]) + return selected + + +def _normalized_sources(sources: Iterable[str]) -> Set[str]: + normalized = {SOURCE_REPLACEMENTS.get(source, source) for source in sources} + normalized.difference_update( + { + "mongo/util/exception_filter_win32.cpp", + "mongo/util/signal_win32.cpp", + "mongo/util/winutil.cpp", + "third_party/shim_intel_decimal128.cppshim_intel_decimal128.cpp", + } + ) + if "third_party/shim_intel_decimal128.cppshim_intel_decimal128.cpp" in sources: + normalized.add("third_party/shim_intel_decimal128.cpp") + return {source for source in normalized if source.endswith((".c", ".cc", ".cpp"))} + + +def select_minimal_tests( + targets: Dict[str, Target], production_targets: Set[str] +) -> tuple[List[Dict[str, Any]], Set[str]]: + """Select tests for the retained single-node modules and their shared test helpers.""" + # Production drops alternative engine registrations, but the MongoD test fixture + # explicitly starts ephemeralForTest. Restore that edge only in the test graph. + targets = dict(targets) + fixture_name = "mongo/db/service_context_d_test_fixture" + targets[fixture_name] = { + **targets[fixture_name], + "dependencies": targets[fixture_name]["dependencies"] + [ + "mongo/db/storage/ephemeral_for_test/storage_ephemeral_for_test"], + } + production_sources = _normalized_sources( + source for name in production_targets for source in targets[name]["sources"] + ) + production_directories = { + PurePosixPath(source).parent.as_posix() for source in production_sources + } + tests: List[Dict[str, Any]] = [] + support_targets: Set[str] = set() + # String-named initialization edges cannot pull an otherwise unreferenced object out of a + # static archive. Retain these providers directly, but only for their original test closure. + initializer_sources = { + # These providers run through named initializers, not references from the test main. + # Retain them only when their original SCons dependency closure includes them. + "mongo/util/options_parser/options_parser_init.cpp", + "mongo/util/version_impl.cpp", + "mongo/util/icu_init.cpp", + "mongo/db/auth/authorization_manager_global.cpp", + # The global constructor calls these factories through MONGO_REGISTER_SHIM, not + # through a linker reference to the implementation classes. Retain both providers + # for query/service-context tests as well as tests that use the classes directly. + "mongo/db/auth/authorization_manager_impl.cpp", + "mongo/db/auth/authorization_session_impl.cpp", + "mongo/db/catalog/database_holder_impl.cpp", + "mongo/db/catalog/database_impl.cpp", + "mongo/db/catalog/collection_impl.cpp", + "mongo/db/catalog/collection_info_cache_impl.cpp", + "mongo/db/catalog/index_catalog_entry_impl.cpp", + "mongo/db/catalog/index_catalog_impl.cpp", + "mongo/db/catalog/index_create_impl.cpp", + } + # Aggregation factories are looked up by stage/accumulator name, not by a C++ symbol. + # As with SCons' library retention, keep their implementation objects when the original + # test closure includes them; otherwise authorization/parser tests silently lose stages. + initializer_sources.update( + source for source in production_sources + if source.startswith(("mongo/db/pipeline/document_source_", + "mongo/db/pipeline/accumulator_", + "mongo/db/pipeline/granularity_rounder_")) + ) + + for name, target in sorted(targets.items()): + if target["kind"] not in ("CppUnitTest", "CppIntegrationTest"): + continue + if name in EXCLUDED_DISTRIBUTED_TEST_TARGETS: + continue + if any(name.startswith(prefix) for prefix in EXCLUDED_TEST_TARGET_PREFIXES): + continue + + main_target = ( + "mongo/unittest/unittest_main" + if target["kind"] == "CppUnitTest" + else "mongo/unittest/integration_test_main" + ) + explicit_dependencies = [ + dependency for dependency in target["dependencies"] if dependency != main_target + ] + dependency_closure = _dependency_closure(targets, target["dependencies"]) + related_closure = _dependency_closure(targets, explicit_dependencies) or set() + if dependency_closure is None: + continue + if any( + dependency.startswith(prefix) + for dependency in dependency_closure + for prefix in FORBIDDEN_TARGET_PREFIXES + ): + continue + # A new third-party dependency would require feature-specific flags and headers outside the + # server profile. Keep tests whose vendor dependencies are already part of the server. + if any( + dependency.startswith("third_party/") and dependency not in production_targets + for dependency in dependency_closure + ): + continue + + test_sources = _normalized_sources(target["sources"]) + source_directories = { + PurePosixPath(source).parent.as_posix() for source in test_sources + } + related = bool(related_closure & production_targets) or bool( + source_directories & production_directories + ) + if not related: + continue + if any( + source.startswith(prefix) + for source in test_sources + for prefix in EXCLUDED_TEST_TARGET_PREFIXES + ): + continue + + test_support = dependency_closure - production_targets - {main_target} + if any(targets[item]["kind"] != "Library" for item in test_support): + continue + per_test_support_sources = _normalized_sources( + source for item in test_support for source in targets[item]["sources"] + ) - production_sources + per_test_support_sources.update( + source for item in dependency_closure for source in targets[item]["sources"] + if source in initializer_sources + ) + if name == "mongo/db/pipeline/document_source_facet_test": + # The facet parser tests use this internal stage to express conflicting + # host requirements. It is a test fixture, not a server registration. + per_test_support_sources.add( + "mongo/db/pipeline/document_source_internal_split_pipeline.cpp") + if any( + source.startswith(prefix) + for source in per_test_support_sources + for prefix in EXCLUDED_TEST_TARGET_PREFIXES + ): + continue + support_targets.update(test_support) + tests.append( + { + "id": hashlib.sha1(name.encode("utf-8")).hexdigest()[:12], + "kind": "unit" if target["kind"] == "CppUnitTest" else "integration", + "name": name, + "requires_eloq": name.startswith("mongo/db/modules/eloq/") + or any(item.startswith("mongo/db/modules/eloq/") for item in dependency_closure), + "sources": sorted(test_sources), + "support_sources": sorted(per_test_support_sources), + } + ) + + support_sources = _normalized_sources( + source for name in support_targets for source in targets[name]["sources"] + ) - production_sources + support_sources.update(source for test in tests for source in test["support_sources"]) + return tests, support_sources + + +def _cmake_list(name: str, values: Iterable[str]) -> str: + body = "\n".join(f' "{value}"' for value in sorted(set(values))) + return f"set({name}\n{body}\n)\n" + + +def _cmake_scalar(name: str, value: str) -> str: + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'set({name} "{escaped}")\n' + + +def generate(source_root: Path, target_arch: str = platform.machine()) -> str: + targets = load_targets(source_root, target_arch) + selected = select_minimal_graph(targets, source_root, target_arch) + sources = _normalized_sources( + source + for name in selected + for source in targets[name]["sources"] + ) + tests, test_support_sources = select_minimal_tests(targets, selected) + all_test_sources = test_support_sources | { + source for test in tests for source in test["sources"] + } + + missing = sorted( + source + for source in sources + if "_gen.cpp" not in source + and source + not in GENERATED_WITHOUT_SOURCE_FILE + and not (source_root / source).is_file() + ) + if missing: + raise RuntimeError("Selected source files do not exist: " + ", ".join(missing)) + + missing_test_sources = sorted( + source + for source in all_test_sources + if "_gen.cpp" not in source + and source not in GENERATED_WITHOUT_SOURCE_FILE + and not (source_root / source).is_file() + ) + if missing_test_sources: + raise RuntimeError( + "Selected test source files do not exist: " + ", ".join(missing_test_sources) + ) + + mongo_sources = [source for source in sources if source.startswith("mongo/")] + third_party_sources = [source for source in sources if source.startswith("third_party/")] + manifest = ( + "# Generated by cmake/generate_eloqdoc_sources.py; do not edit.\n" + + _cmake_list("ELOQDOC_MONGO_TARGETS", selected) + + _cmake_list("ELOQDOC_MONGO_SOURCES", mongo_sources) + + _cmake_list("ELOQDOC_THIRD_PARTY_SOURCES", third_party_sources) + + _cmake_list("ELOQDOC_MONGO_TEST_SUPPORT_SOURCES", test_support_sources) + + _cmake_list("ELOQDOC_MONGO_TEST_IDS", (test["id"] for test in tests)) + ) + for test in tests: + prefix = f'ELOQDOC_MONGO_TEST_{test["id"]}' + manifest += _cmake_scalar(prefix + "_NAME", test["name"]) + manifest += _cmake_scalar(prefix + "_KIND", test["kind"]) + manifest += _cmake_scalar( + prefix + "_REQUIRES_ELOQ", "TRUE" if test["requires_eloq"] else "FALSE" + ) + manifest += _cmake_list(prefix + "_SOURCES", test["sources"]) + manifest += _cmake_list(prefix + "_SUPPORT_SOURCES", test["support_sources"]) + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--source-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--target-arch", default=platform.machine()) + args = parser.parse_args() + args.output.parent.mkdir(parents=True, exist_ok=True) + manifest = generate(args.source_root.resolve(), args.target_arch) + if not args.output.exists() or args.output.read_text(encoding="utf-8") != manifest: + args.output.write_text(manifest, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/cmake/generate_error_codes.py b/cmake/generate_error_codes.py new file mode 100644 index 0000000000..686c558e10 --- /dev/null +++ b/cmake/generate_error_codes.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Generate MongoDB error-code files without changing its Python 2/Cheetah generator. + +MongoDB 4.0's SCons build continues to invoke src/mongo/base/generate_error_codes.py. +The CMake build uses this Python 3-only renderer for the small Cheetah directive subset present in +the checked-in error-code templates. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path +from typing import Any, Dict, Iterable, List, Sequence, Tuple + + +DIRECTIVE = re.compile(r"^\s*//#(for|if|set)\s+(.*?)\s*$") +END_DIRECTIVE = re.compile(r"^\s*//#end\s+(for|if)\s*$") +BRACED_VALUE = re.compile(r"\$\{([^}]+)\}") +VALUE = re.compile(r"\$([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)") + + +class ErrorCode: + def __init__(self, name: str, code: int, extra: str | None = None): + self.name = name + self.code = code + self.extra = extra + self.categories: List[str] = [] + + +class ErrorClass: + def __init__(self, name: str, codes: Sequence[str]): + self.name = name + self.codes = codes + + +def parse_definitions(path: Path) -> Tuple[List[ErrorCode], List[ErrorClass]]: + error_codes: List[ErrorCode] = [] + error_classes: List[ErrorClass] = [] + namespace = { + "error_code": lambda *args, **kwargs: error_codes.append(ErrorCode(*args, **kwargs)), + "error_class": lambda *args: error_classes.append(ErrorClass(*args)), + } + source = path.read_text(encoding="utf-8") + exec(compile(source, str(path), "exec"), namespace) + error_codes.sort(key=lambda item: item.code) + + names: Dict[str, ErrorCode] = {} + numeric_codes: Dict[int, ErrorCode] = {} + for error_code in error_codes: + if error_code.name in names: + raise ValueError(f"Duplicate error-code name: {error_code.name}") + if error_code.code in numeric_codes: + raise ValueError(f"Duplicate numeric error code: {error_code.code}") + names[error_code.name] = error_code + numeric_codes[error_code.code] = error_code + + category_names = set() + for category in error_classes: + if category.name in category_names: + raise ValueError(f"Duplicate error category: {category.name}") + category_names.add(category.name) + for error_name in category.codes: + if error_name not in names: + raise ValueError( + f"Undeclared error code {error_name} in category {category.name}" + ) + names[error_name].categories.append(category.name) + return error_codes, error_classes + + +def evaluate(expression: str, values: Dict[str, Any]) -> Any: + expression = re.sub(r"\$([A-Za-z_]\w*)", r"\1", expression).rstrip(":") + scope: Dict[str, Any] = {"__builtins__": {}, "enumerate": enumerate, "len": len} + scope.update(values) + return eval(expression, scope) + + +def parse_template( + lines: Sequence[str], position: int = 0, terminator: str | None = None +) -> Tuple[List[tuple], int]: + nodes: List[tuple] = [] + while position < len(lines): + end_match = END_DIRECTIVE.match(lines[position]) + if end_match: + if end_match.group(1) != terminator: + raise ValueError(f"Unexpected template directive: {lines[position].strip()}") + return nodes, position + 1 + + match = DIRECTIVE.match(lines[position]) + if not match: + nodes.append(("text", lines[position])) + position += 1 + continue + + operation, argument = match.groups() + if operation == "for": + target, expression = argument.split(" in ", 1) + names = [name.strip().lstrip("$") for name in target.split(",")] + body, position = parse_template(lines, position + 1, "for") + nodes.append(("for", names, expression, body)) + elif operation == "if": + body, position = parse_template(lines, position + 1, "if") + nodes.append(("if", argument, body)) + else: + name, expression = argument.split("=", 1) + nodes.append(("set", name.strip().lstrip("$"), expression.strip())) + position += 1 + + if terminator: + raise ValueError(f"Missing //#end {terminator}") + return nodes, position + + +def substitute(line: str, values: Dict[str, Any]) -> str: + def replace(match: re.Match[str]) -> str: + return str(evaluate(match.group(1), values)) + + return VALUE.sub(replace, BRACED_VALUE.sub(replace, line)) + + +def render_nodes(nodes: Iterable[tuple], values: Dict[str, Any]) -> List[str]: + result: List[str] = [] + for node in nodes: + if node[0] == "text": + result.append(substitute(node[1], values)) + elif node[0] == "set": + values[node[1]] = evaluate(node[2], values) + elif node[0] == "if": + if evaluate(node[1], values): + result.extend(render_nodes(node[2], dict(values))) + else: + _, names, expression, body = node + for item in evaluate(expression, values): + nested = dict(values) + unpacked = item if len(names) > 1 else (item,) + nested.update(zip(names, unpacked)) + result.extend(render_nodes(body, nested)) + return result + + +def render_template(path: Path, values: Dict[str, Any]) -> str: + nodes, _ = parse_template(path.read_text(encoding="utf-8").splitlines(keepends=True)) + return "".join(render_nodes(nodes, values)) + + +def main(argv: Sequence[str]) -> int: + if len(argv) < 3: + raise SystemExit( + "usage: generate_error_codes.py error_codes.err template=output [template=output ...]" + ) + error_codes, error_classes = parse_definitions(Path(argv[1])) + values = {"codes": error_codes, "categories": error_classes} + for argument in argv[2:]: + template, output = argument.split("=", 1) + Path(output).write_text(render_template(Path(template), values), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/cmake/generate_icu_init.py b/cmake/generate_icu_init.py new file mode 100644 index 0000000000..58d439899e --- /dev/null +++ b/cmake/generate_icu_init.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Run SCons' unchanged ICU-data generator with Python 3 byte/text semantics.""" + +import builtins +import runpy +import sys + + +def generator_open(path, mode="r", *args, **kwargs): + # Keep the .dat input binary, but Python 2's generated C++ output is a str. + if mode == "wb": + return builtins.open(path, "w", encoding="utf-8", newline="\n") + return builtins.open(path, mode, *args, **kwargs) + + +def byte_value(value): + return value if isinstance(value, int) else ord(value) + + +def main(): + if len(sys.argv) < 2: + raise SystemExit("usage: generate_icu_init.py SCRIPT -i DATA -o OUTPUT") + sys.argv = sys.argv[1:] + runpy.run_path(sys.argv[0], run_name="__main__", + init_globals={"open": generator_open, "ord": byte_value}) + + +if __name__ == "__main__": + main() diff --git a/cmake/generate_idl.py b/cmake/generate_idl.py new file mode 100644 index 0000000000..2973382ae5 --- /dev/null +++ b/cmake/generate_idl.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Run MongoDB's Python 2 IDL compiler from the CMake/Python 3 build. + +The historical SCons build invokes scripts/buildscripts/idl/idlc.py with +Python 2. Keep that implementation untouched and provide the two small +Python 3 compatibility shims here so the build systems do not affect one +another. +""" + +import argparse +import builtins +import itertools +import sys +from pathlib import Path + + +def escape_depfile_path(path): + """Escape a path for Ninja/CMake's Make-style depfile parser.""" + return (str(path).replace("\\", "\\\\").replace("$", "$$") + .replace("#", "\\#").replace(" ", "\\ ").replace(":", "\\:")) + + +def enable_depfile(compiler, depfile): + """Record the imports actually opened by the unchanged SCons IDL compiler.""" + dependencies = set() + original_compile = compiler.compile_idl + original_open = compiler.CompilerImportResolver.open + + def tracking_open(self, resolved_file_name): + dependencies.add(Path(resolved_file_name).resolve()) + return original_open(self, resolved_file_name) + + def compile_with_dependencies(args): + dependencies.clear() + dependencies.add(Path(args.input_file).resolve()) + if not original_compile(args): + return False + source = args.output_source or str(Path(args.input_file).with_suffix("")) + "_gen.cpp" + depfile.write_text( + escape_depfile_path(Path(source).resolve()) + ": " + + " ".join(escape_depfile_path(path) for path in sorted(dependencies)) + "\n", + encoding="utf-8") + return True + + compiler.CompilerImportResolver.open = tracking_open + compiler.compile_idl = compile_with_dependencies + + +def main(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--depfile", type=Path) + args, compiler_args = parser.parse_known_args() + sys.argv[1:] = compiler_args + + idl_root = Path(__file__).resolve().parents[1] / "scripts" / "buildscripts" / "idl" + sys.path.insert(0, str(idl_root)) + + # binder.py uses Python 2's xrange. + builtins.xrange = range + + import idlc # pylint: disable=import-error,import-outside-toplevel + from idl import compiler, syntax # pylint: disable=import-error,import-outside-toplevel + + # syntax.py uses dict.viewitems(), which has no Python 3 equivalent. + def item_and_type(values): + return itertools.chain.from_iterable( + syntax._zip_scalar(value, key) for key, value in values.items()) + + syntax._item_and_type = item_and_type + if args.depfile: + enable_depfile(compiler, args.depfile) + idlc.main() + + +if __name__ == "__main__": + main() diff --git a/cmake/generate_js.py b/cmake/generate_js.py new file mode 100644 index 0000000000..469f903bf3 --- /dev/null +++ b/cmake/generate_js.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Execute the unchanged SCons JavaScript embedding generator using Python 3.""" + +import sys +from pathlib import Path + + +def main(): + script = Path(sys.argv[1]) + source = script.read_text(encoding="utf-8") + replacements = { + 'print "Must specify [target] [source] "': + 'print("Must specify [target] [source] ")', + "out.write(text)": "out.write(text.encode('utf-8'))", + } + for old, new in replacements.items(): + if source.count(old) != 1: + raise RuntimeError("unexpected jstoh.py format: " + old) + source = source.replace(old, new) + sys.argv = sys.argv[1:] + exec(compile(source, str(script), "exec"), {"__file__": str(script), "__name__": "__main__"}) + + +if __name__ == "__main__": + main() diff --git a/cmake/run_legacy_text_generator.py b/cmake/run_legacy_text_generator.py new file mode 100644 index 0000000000..04a39ebb14 --- /dev/null +++ b/cmake/run_legacy_text_generator.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Run simple Python 2 text generators under Python 3. + +These generators use binary output modes for strings or the removed ``U`` +mode. CMake needs Python 3, while SCons must keep executing the original +files with Python 2 and their historical I/O behavior. +""" + +import builtins +import runpy +import sys +from pathlib import Path + + +def main(): + if len(sys.argv) < 2: + raise SystemExit("usage: run_legacy_text_generator.py SCRIPT [ARG ...]") + + script = sys.argv[1] + sys.argv = sys.argv[1:] + sys.path.insert(0, str(Path(script).resolve().parent)) + original_open = builtins.open + + def text_open(path, mode="r", *args, **kwargs): + mode = mode.replace("b", "").replace("U", "") or "r" + return original_open(path, mode, *args, **kwargs) + + builtins.open = text_open + runpy.run_path(script, run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/cmake/tests/server_smoke.py b/cmake/tests/server_smoke.py new file mode 100644 index 0000000000..727c9643ea --- /dev/null +++ b/cmake/tests/server_smoke.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Exercise the real CMake server over the wire (requires pymongo>=4.6,<4.11). + +Uses a private Data Substrate fixture with two worker cores and a disposable local data +store (WAL disabled). Build with ELOQDSS_ROCKSDB/ROCKSDB, not a cloud backend. Logs and +data are retained in the printed temporary directory. +""" + +import argparse +import datetime +import json +import os +from pathlib import Path +import socket +import signal +import subprocess +import tempfile +import threading +import time + +from pymongo import MongoClient +from bson.int64 import Int64 +from pymongo.errors import AutoReconnect, OperationFailure, PyMongoError +from pymongo.read_concern import ReadConcern + + +def free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def eventually(predicate, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.25) + raise AssertionError("condition did not become true within %ss" % timeout) + + +def command_fails(database, command, *, code=None, message): + try: + database.command(command) + except OperationFailure as exc: + if code is not None: + assert exc.code == code, exc + assert message in str(exc), exc + else: + raise AssertionError("command unexpectedly succeeded: " + repr(command)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--server", required=True, type=Path) + args = parser.parse_args() + server = args.server.resolve() + # Keep every fixture child, including auxiliary services, within the user's CPU limit. + if hasattr(os, "sched_getaffinity"): + os.sched_setaffinity(0, sorted(os.sched_getaffinity(0))[:8]) + root = Path(tempfile.mkdtemp(prefix="eloqdoc-server-smoke-")) + print("Fixture:", root, flush=True) + (root / "db").mkdir() + port, tx_port = free_port(), free_port() + config = root / "data_substrate.cnf" + config.write_text( + "[local]\ncore_number=2\nnode_memory_limit_mb=512\n" + "enable_data_store=true\nenable_wal=false\nenable_mvcc=true\n" + "event_dispatcher_num=1\nbind_all=false\n" + f"tx_ip=127.0.0.1\ntx_port={tx_port}\nhm_port={free_port()}\n" + f"eloq_data_path={root / 'eloq'}\nlog_service_data_path={root / 'log'}\n" + f"[cluster]\ntx_ip_port_list=127.0.0.1:{tx_port}\n", + encoding="utf-8") + server_config = root / "eloqdoc.yaml" + # JSON is valid YAML and avoids adding another test-client dependency. + server_config.write_text(json.dumps({ + "storage": {"dbPath": str(root / "db")}, + "net": {"port": port, "bindIp": "127.0.0.1", + "serviceExecutor": "adaptive", "adaptiveThreadNum": 1}, + }), encoding="utf-8") + common = [str(server), "--data_substrate_config=" + str(config), + "--config=" + str(server_config), "--log_dir=" + str(root)] + for option in ("--replSet=test", "--shardsvr", "--configsvr", "--storageEngine=wiredTiger"): + result = subprocess.run(common + [option], cwd=root, capture_output=True, timeout=30) + (root / (option.split("=")[0][2:] + ".log")).write_bytes(result.stdout + result.stderr) + assert result.returncode > 0, option + " was accepted or crashed instead of being rejected" + expected = (b"unknown storage engine: wiredTiger" if "storageEngine" in option + else b"supports standalone mode only") + assert expected in result.stdout + result.stderr, "unexpected failure for " + option + print("PASS unsupported topology/storage options rejected", flush=True) + + with (root / "server.log").open("wb") as log: + process = subprocess.Popen(common + ["--auth", "--setParameter=ttlMonitorSleepSecs=1", + # Exercise the two test-only maintenance-mode commands too. + "--setParameter=enableTestCommands=1", + "--setParameter=diagnosticDataCollectionEnabled=false"], + cwd=root, stdout=log, stderr=subprocess.STDOUT, + start_new_session=True) + anonymous = MongoClient("127.0.0.1", port, directConnection=True, + serverSelectionTimeoutMS=1000, socketTimeoutMS=15000) + authenticated = None + try: + last_connection_error = [] + def ready(): + assert process.poll() is None, "server exited; see " + str(root / "server.log") + try: + return anonymous.admin.command("ping")["ok"] == 1 + except PyMongoError as exc: + last_connection_error[:] = [str(exc)] + return False + try: + eventually(ready, 90) + except AssertionError as exc: + raise AssertionError(f"{exc}; last client error: {last_connection_error}") from exc + hello = anonymous.admin.command("ismaster") + assert hello["ismaster"] and "setName" not in hello + anonymous.admin.command("createUser", "smoke_root", pwd="temporary-smoke-password", + roles=["root"]) + authenticated = MongoClient("127.0.0.1", port, directConnection=True, + username="smoke_root", password="temporary-smoke-password", + authSource="admin", serverSelectionTimeoutMS=5000, + socketTimeoutMS=15000) + db = authenticated.cmake_smoke + try: + anonymous.cmake_smoke.records.insert_one({"unauthorized": True}) + raise AssertionError("unauthenticated write was permitted with --auth") + except OperationFailure as exc: + assert exc.code == 13, exc + print("PASS authentication and authorization", flush=True) + commands = authenticated.admin.command("listCommands")["commands"] + for command in ("isMaster", "find", "insert", "aggregate", "mapReduce", "createUser", + "createIndexes", "shutdown", "fsync", "compact", "touch", + "repairDatabase", "invalidateViewCatalog", "restartCatalog"): + assert command in commands, "missing standalone command: " + command + for command in ("replSetInitiate", "replSetReconfig", "_configsvrAddShard", + "mapreduce.shardedfinish", "shardConnPoolStats"): + assert command not in commands, "distributed command retained: " + command + print("PASS standalone command registration", flush=True) + db.records.insert_many([{"k": i, "v": i % 2} for i in range(10)]) + db.records.create_index("k", unique=True, background=True) + assert db.records.find_one({"k": 3})["v"] == 1 + db.records.update_one({"k": 3}, {"$set": {"v": 7}}) + assert db.records.find_one({"k": 3})["v"] == 7 + rows = list(db.records.aggregate([{"$group": {"_id": None, "n": {"$sum": 1}}}])) + assert rows[0]["n"] == 10 + db.records.delete_one({"k": 9}) + assert db.records.count_documents({}) == 9 + print("PASS CRUD, background index, local aggregation", flush=True) + # Collation metadata must remain compatible with the historical SCons build. + collation = {"locale": "en_US", "strength": 2, "version": "57.1"} + db.collated.create_index("s", collation=collation) + db.collated.insert_one({"s": "Apple"}) + assert db.collated.find_one({"s": "APPLE"}, collation=collation)["s"] == "Apple" + index = next(item for item in db.collated.list_indexes() if item["name"] == "s_1") + assert index["collation"]["version"] == "57.1", index + print("PASS SCons-compatible ICU 57.1 collation metadata and lookup", flush=True) + # These must reach their handlers, which report the existing engine restrictions, + # instead of throwing NoReplicationEnabled from MaintenanceModeSetter. + command_fails(db, {"compact": "records"}, code=115, message="cannot compact") + command_fails(db, {"touch": "records", "data": True}, code=115, + message="does not support touch") + command_fails(db, {"repairDatabase": 1}, + message="Eloq storage engine does not require manual database repair") + db.command("create", "records_view", viewOn="records", pipeline=[]) + db.command("invalidateViewCatalog", 1) + assert db.records_view.count_documents({}) == 9 + print("PASS maintenance-mode handlers and view catalog reload", flush=True) + # getMore with a term requires internal authorization even on a standalone server. + authenticated.admin.command("createUser", "smoke_internal", + pwd="temporary-internal-password", roles=["__system"]) + with MongoClient("127.0.0.1", port, directConnection=True, + username="smoke_internal", password="temporary-internal-password", + authSource="admin", serverSelectionTimeoutMS=5000, + socketTimeoutMS=15000) as internal: + cursor = internal.cmake_smoke.command("find", "records", batchSize=1)["cursor"] + assert cursor["id"] != 0 + try: + command_fails(internal.cmake_smoke, + {"getMore": cursor["id"], "collection": "records", "term": Int64(1)}, + code=2, message="cannot supply 'term' without active replication") + finally: + internal.cmake_smoke.command("killCursors", "records", cursors=[cursor["id"]]) + authenticated.admin.command("dropUser", "smoke_internal") + assert authenticated.admin.command("getParameter", 1, replIndexPrefetch=1)[ + "replIndexPrefetch"] == "uninitialized" + command_fails(authenticated.admin, {"setParameter": 1, "replIndexPrefetch": "none"}, + code=2, message="replication is not enabled") + print("PASS getMore term and replication parameter status codes", flush=True) + assert len(list(db.records.find({"$where": "function() { return this.k === 2; }"}))) == 1 + print("PASS server-side JavaScript", flush=True) + # The existing Eloq command explicitly disallows map-reduce, independently of JS. + try: + db.command("mapReduce", "records", + map="function() { emit(this.v, 1); }", + reduce="function(key, values) { return Array.sum(values); }", + out={"inline": 1}) + raise AssertionError("existing Eloq map-reduce restriction changed") + except OperationFailure as exc: + assert "Eloq storage engine does not support mapreduce" in str(exc), exc + print("PASS existing Eloq map-reduce restriction preserved", flush=True) + db.expiring.create_index("expires", expireAfterSeconds=0) + db.expiring.insert_one({"expires": datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(minutes=5)}) + eventually(lambda: db.expiring.count_documents({}) == 0, 30) + print("PASS TTL deletion by background worker", flush=True) + with authenticated.start_session() as session: + assert db.command("ping", session=session)["ok"] == 1 + with session.start_transaction(read_concern=ReadConcern("local")): + db.records.update_one({"k": 0}, {"$set": {"v": 20}}, session=session) + db.records.update_one({"k": 1}, {"$set": {"v": 21}}, session=session) + assert db.records.find_one({"k": 0})["v"] == 20 + assert db.records.find_one({"k": 1})["v"] == 21 + session.start_transaction(read_concern=ReadConcern("local")) + db.records.update_one({"k": 0}, {"$set": {"v": -1}}, session=session) + session.abort_transaction() + assert db.records.find_one({"k": 0})["v"] == 20 + # Check the normal background refresh, without invoking refreshLogicalSessionCacheNow. + eventually(lambda: "lsidTTLIndex" in + authenticated.config.system.sessions.index_information(), 30) + print("PASS logical sessions, transaction commit/abort, and maintenance", flush=True) + authenticated.admin.command("fsync") + authenticated.admin.command({"fsync": 1, "async": True}) + # Exercise the lock thread's flush and backup path as well. Eloq's KV engine does + # not implement backup mode, so the normal storage-engine error must survive. + command_fails(authenticated.admin, {"fsync": 1, "lock": True}, code=115, + message="doesn't support backup mode") + status = authenticated.admin.command("serverStatus") + assert status["storageEngine"]["name"] == "eloq" + assert "setName" not in status.get("repl", {}) + print("PASS fsync and server diagnostics", flush=True) + for _ in range(2): + # Eloq's no-op global locker cannot protect a live catalog reload across workers. + # Reject safely before invalidating any catalog, then verify the server still works. + command_fails(authenticated.admin, {"restartCatalog": 1}, code=115, + message="does not support online catalog restart") + assert db.records.count_documents({}) == 9 + assert db.records_view.count_documents({}) == 9 + assert db.records.find_one({"k": 0})["v"] == 20 + print("PASS unsafe online catalog restart rejected without affecting data", flush=True) + # The network integration suites can leave a long-running command behind. Shutdown + # must interrupt it, not wait for its normal completion or require SIGTERM. + sleep_finished = threading.Event() + sleep_errors = [] + def sleeping_command(): + try: + authenticated.admin.command({"sleep": 1, "secs": 600, "lock": "none", + "comment": "shutdown-smoke"}) + sleep_errors.append("sleep completed before shutdown") + except AutoReconnect: + pass + except OperationFailure as exc: + if exc.code not in (91, 11600): # ShutdownInProgress, InterruptedAtShutdown + sleep_errors.append(str(exc)) + except Exception as exc: + sleep_errors.append(str(exc)) + finally: + sleep_finished.set() + sleeper = threading.Thread(target=sleeping_command, daemon=True) + sleeper.start() + def sleep_is_active(): + assert not sleep_finished.is_set(), sleep_errors + return any(op.get("command", {}).get("comment") == "shutdown-smoke" + for op in authenticated.admin.command("currentOp")["inprog"]) + eventually(sleep_is_active, 10) + try: + authenticated.admin.command("shutdown", force=True) + except AutoReconnect: + pass + assert process.wait(timeout=30) == 0 + assert sleep_finished.wait(5), "sleep command did not finish during shutdown" + assert not sleep_errors, sleep_errors + print("PASS clean shutdown with an active command", flush=True) + finally: + anonymous.close() + if authenticated: + authenticated.close() + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + # The fixture owns this new session. Also stop auxiliary children after a crash. + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + + +if __name__ == "__main__": + main() diff --git a/cmake/tests/test_icu_generator.py b/cmake/tests/test_icu_generator.py new file mode 100644 index 0000000000..cc675714ac --- /dev/null +++ b/cmake/tests/test_icu_generator.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Check the Python 3 adapter preserves the original ICU data and initializer.""" + +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest + + +class ICUGeneratorTest(unittest.TestCase): + def test_all_byte_values_are_embedded_unchanged(self): + root = Path(__file__).resolve().parents[2] + with tempfile.TemporaryDirectory() as directory: + data = Path(directory) / "data.dat" + output = Path(directory) / "icu_init.cpp" + payload = bytes(range(256)) + b"\r\n\x00\xff" + data.write_bytes(payload) + subprocess.run([ + sys.executable, str(root / "cmake/generate_icu_init.py"), + str(root / "src/mongo/util/generate_icu_init_cpp.py"), + "-i", str(data), "-o", str(output)], check=True) + source = output.read_text() + encoded = re.search(r"kRawData\[\] = \{([\d,]+)\};", source) + self.assertIsNotNone(encoded) + self.assertEqual(payload, bytes(int(value) for value in encoded.group(1).split(","))) + self.assertIn("alignas(16)", source) + self.assertIn("MONGO_INITIALIZER(LoadICUData)", source) + self.assertIn("udata_setCommonData(kRawData, &status)", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/tests/test_idl_dependencies.py b/cmake/tests/test_idl_dependencies.py new file mode 100644 index 0000000000..882c4eea44 --- /dev/null +++ b/cmake/tests/test_idl_dependencies.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Exercise the real IDL compiler and Ninja's transitive dependency tracking.""" + +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from generate_idl import escape_depfile_path + + +class IDLDependenciesTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="eloqdoc idl ") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.build = self.root / "build" + self.wrapper = Path(__file__).resolve().parents[1] / "generate_idl.py" + leaf = '''types: + testString: + description: A test string + cpp_type: std::string + bson_serialization_type: string + deserializer: mongo::BSONElement::str +''' + (self.root / "leaf.idl").write_text(leaf, encoding="utf-8") + (self.root / "other.idl").write_text(leaf, encoding="utf-8") + (self.root / "middle.idl").write_text("imports: [leaf.idl]\n", encoding="utf-8") + for name, imported in (("first", "middle.idl"), ("second", "other.idl")): + (self.root / (name + ".idl")).write_text( + "imports: [" + imported + "]\n" + "global:\n cpp_namespace: mongo\n" + "structs:\n Example:\n description: Test struct\n" + " fields:\n value: testString\n", encoding="utf-8") + (self.root / "CMakeLists.txt").write_text(''' +cmake_minimum_required(VERSION 3.24) +project(IDLDependencies NONE) +foreach(name IN ITEMS first second) + set(output "${CMAKE_CURRENT_BINARY_DIR}/${name}_gen.cpp") + add_custom_command( + OUTPUT "${output}" "${CMAKE_CURRENT_BINARY_DIR}/${name}_gen.h" + COMMAND "${IDL_PYTHON}" "${IDL_WRAPPER}" + --include "${CMAKE_CURRENT_SOURCE_DIR}" + --base_dir "${CMAKE_CURRENT_BINARY_DIR}" + --output "${output}" + --header "${CMAKE_CURRENT_BINARY_DIR}/${name}_gen.h" + --depfile "${output}.d" + "${CMAKE_CURRENT_SOURCE_DIR}/${name}.idl" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${name}.idl" "${IDL_WRAPPER}" + DEPFILE "${output}.d" + VERBATIM) + add_custom_target(${name} ALL DEPENDS "${output}") +endforeach() +''', encoding="utf-8") + self.run_command("cmake", "-S", self.root, "-B", self.build, "-G", "Ninja", + "-DIDL_PYTHON=" + sys.executable, "-DIDL_WRAPPER=" + str(self.wrapper)) + + def run_command(self, *args): + result = subprocess.run([str(arg) for arg in args], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return result.stdout + + def build_outputs(self): + self.run_command("cmake", "--build", self.build, "-j2") + return tuple((self.build / (name + "_gen.cpp")).stat().st_mtime_ns + for name in ("first", "second")) + + def test_depfile_lists_only_actual_transitive_inputs(self): + self.build_outputs() + depfile = (self.build / "first_gen.cpp.d").read_text(encoding="utf-8") + for name in ("first.idl", "middle.idl", "leaf.idl"): + self.assertIn(escape_depfile_path(self.root / name), depfile) + self.assertNotIn("other.idl", depfile) + self.assertNotIn("second.idl", depfile) + + def test_incremental_build_tracks_changed_and_new_imports(self): + original = self.build_outputs() + self.assertEqual(original, self.build_outputs(), "no-op build regenerated IDL") + with (self.root / "leaf.idl").open("a", encoding="utf-8") as stream: + stream.write("\n# Changed transitive dependency\n") + changed = self.build_outputs() + self.assertNotEqual(original[0], changed[0]) + self.assertEqual(original[1], changed[1], "unrelated output was regenerated") + + (self.root / "new.idl").write_text("global:\n cpp_namespace: mongo\n", + encoding="utf-8") + (self.root / "middle.idl").write_text("imports: [leaf.idl, new.idl]\n", + encoding="utf-8") + imported = self.build_outputs() + self.assertNotEqual(changed[0], imported[0]) + self.assertEqual(changed[1], imported[1]) + with (self.root / "new.idl").open("a", encoding="utf-8") as stream: + stream.write("\n# Newly discovered dependency changed\n") + updated = self.build_outputs() + self.assertNotEqual(imported[0], updated[0]) + self.assertEqual(imported[1], updated[1]) + self.assertEqual(updated, self.build_outputs()) + + def test_failed_compilation_does_not_publish_depfile(self): + (self.root / "first.idl").write_text("imports: [missing.idl]\n", encoding="utf-8") + result = subprocess.run(["cmake", "--build", str(self.build), "--target", "first", "-j2"], + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertFalse((self.build / "first_gen.cpp.d").exists()) + + def test_depfile_does_not_change_generated_cpp_or_header(self): + self.build_outputs() + cpp = self.build / "first_gen.cpp" + header = self.build / "first_gen.h" + expected = (cpp.read_bytes(), header.read_bytes()) + self.run_command(sys.executable, self.wrapper, "--include", self.root, + "--base_dir", self.build, "--output", cpp, "--header", header, + self.root / "first.idl") + self.assertEqual(expected, (cpp.read_bytes(), header.read_bytes())) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/tests/test_scons_test_selection.py b/cmake/tests/test_scons_test_selection.py new file mode 100644 index 0000000000..a5692b118c --- /dev/null +++ b/cmake/tests/test_scons_test_selection.py @@ -0,0 +1,63 @@ +"""Tests for the explicit SCons EloqDoc validation profile (Python 3).""" + +import importlib.util +from pathlib import Path +import unittest + +spec = importlib.util.spec_from_file_location( + "select_eloq_unit_tests", Path(__file__).resolve().parents[2] / + "scripts/buildscripts/select_eloq_unit_tests.py") +selector = importlib.util.module_from_spec(spec) +spec.loader.exec_module(selector) + + +class EloqUnitTestSelectionTest(unittest.TestCase): + def test_unsupported_engines_and_embedded_are_skipped(self): + for target in ("mongo/db/storage/mmap_v1/btree/btree_interface_test", + "mongo/db/storage/wiredtiger/storage_wiredtiger_prefixed_record_store_test", + "mongo/embedded/mongo_embedded_capi_test"): + for prefix in ("build/gcc15/", "/tmp/test-build/gcc15/"): + self.assertIsNotNone(selector.skip_reason(prefix + target)) + + def test_obsolete_interfaces_have_narrow_exclusions(self): + self.assertIsNotNone(selector.skip_reason("build/gcc15/mongo/db/views/views_test")) + self.assertIsNone(selector.skip_reason("build/gcc15/mongo/db/pipeline/pipeline_test")) + + def test_replication_suites_are_excluded_including_future_targets(self): + for name in ("replication_process_test", "replication_consistency_markers_impl_test", + "drop_pending_collection_reaper_test", "replication_coordinator_impl_test", + "initial_syncer_test", "future_test"): + for prefix in ("", "build/gcc15/", "/tmp/test-build/gcc15/"): + self.assertIn("replication", selector.skip_reason(prefix + "mongo/db/repl/" + name)) + + def test_fixed_tests_and_applicable_catalog_coverage_remain_selected(self): + for name in ("mongo/db/query/collation/collator_factory_icu_test", + "mongo/db/operation_time_tracker_test", "mongo/util/background_job_test", + "mongo/s/query/async_results_merger_test", + "mongo/db/catalog/collection_test", + "mongo/db/storage/kv/kv_collection_catalog_entry_test", + "mongo/db/storage/storage_repair_observer_test", + "mongo/db/standalone_runtime_test", + "mongo/db/free_mon/free_mon_test", + "mongo/db/storage/ephemeral_for_test/storage_ephemeral_for_test_record_store_test"): + self.assertIsNone(selector.skip_reason(name)) + + def test_confirmed_legacy_fixtures_have_explicit_reasons(self): + for target, reason in selector.UNSUPPORTED_RUNTIME_FIXTURES.items(): + self.assertTrue(reason) + self.assertEqual(reason, selector.skip_reason("/tmp/build/gcc15/" + target)) + self.assertIn("oplog", selector.skip_reason("mongo/db/catalog/database_test")) + self.assertIn("renameCollection", + selector.skip_reason("mongo/db/catalog/create_collection_test")) + self.assertIn("repair", selector.skip_reason("mongo/db/storage/kv/kv_storage_engine_test")) + # New tests in these directories must be assessed, not silently skipped. + self.assertIsNone(selector.skip_reason("mongo/db/catalog/future_test")) + self.assertIsNone(selector.skip_reason("mongo/db/s/future_test")) + + def test_missing_or_unrecognized_binary_is_not_silently_skipped(self): + binaries = ["/no-such-build/mongo/db/future_test", "unknown_test"] + self.assertEqual((binaries, []), selector.select_tests(binaries)) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/tests/test_source_graph.py b/cmake/tests/test_source_graph.py new file mode 100644 index 0000000000..bff60cc134 --- /dev/null +++ b/cmake/tests/test_source_graph.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Regression checks for the standalone server's feature and dependency boundaries.""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from generate_eloqdoc_sources import ( + EXCLUDED_DISTRIBUTED_TEST_TARGETS, load_targets, select_minimal_graph, select_minimal_tests) + + +class StandaloneGraphTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.root = Path(__file__).resolve().parents[2] + cls.targets = load_targets(cls.root / "src") + cls.full_pipeline_sources = set(cls.targets["mongo/db/pipeline/pipeline"]["sources"]) + cls.selected = select_minimal_graph(cls.targets) + cls.sources = { + source for name in cls.selected for source in cls.targets[name]["sources"] + } + + def test_normal_server_features_are_retained(self): + required = { + "mongo/db/dbmain.cpp", "mongo/db/db.cpp", + "mongo/db/service_entry_point_mongod.cpp", + "mongo/db/op_observer_impl.cpp", + "mongo/db/repl/replication_info.cpp", + "mongo/db/auth/authorization_manager_impl.cpp", + "mongo/db/auth/authorization_session_impl.cpp", + "mongo/db/auth/sasl_commands.cpp", "mongo/db/commands/user_management_commands.cpp", + "mongo/db/ttl.cpp", "mongo/db/clientcursor.cpp", + "mongo/db/logical_session_cache_factory_standalone.cpp", + "mongo/db/periodic_runner_job_abort_expired_transactions.cpp", + "mongo/scripting/mozjs/engine.cpp", "mongo/db/commands/mr.cpp", + "mongo/db/commands/shutdown_d.cpp", "mongo/db/commands/fsync.cpp", + "mongo/db/modules/eloq/src/eloq_init.cpp", + } + self.assertFalse(required - self.sources, sorted(required - self.sources)) + + def test_no_embedded_application_or_replica_set_implementation(self): + for source in self.sources: + self.assertFalse(source.startswith("mongo/embedded/"), source) + self.assertNotIn("replication_coordinator_impl", source) + self.assertNotIn("initial_syncer", source) + self.assertNotIn("/bgsync", source) + self.assertNotIn("mongo/scripting/scripting_none.cpp", self.sources) + self.assertNotIn("mongo/s/query/async_results_merger.cpp", self.sources) + self.assertNotIn("mongo/db/pipeline/document_source_merge_cursors.cpp", self.sources) + self.assertNotIn("mongo/client/parallel.cpp", self.sources) + self.assertNotIn("mongo/s/client/shard_connection.cpp", self.sources) + self.assertNotIn("mongo/db/logical_session_cache_factory_mongod.cpp", self.sources) + self.assertNotIn("mongo/db/sessions_collection_sharded.cpp", self.sources) + self.assertNotIn("mongo/db/sessions_collection_rs.cpp", self.sources) + + def test_collations_use_scons_pinned_icu_and_data(self): + self.assertTrue({ + "third_party/icu4c-57.1/source/icu_common", + "third_party/icu4c-57.1/source/icu_i18n", + "third_party/icu4c-57.1/source/icu_data", + } <= self.selected) + self.assertIn("mongo/util/icu_init.cpp", self.sources) + self.assertNotIn("mongo/util/icu_init_stub.cpp", self.sources) + + def test_integration_tests_retain_startup_initializers(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + integration = [test for test in tests if test["kind"] == "integration"] + self.assertEqual(7, len(integration)) + for test in integration: + with self.subTest(name=test["name"]): + self.assertIn("mongo/util/options_parser/options_parser_init.cpp", + test["support_sources"]) + self.assertIn("mongo/util/version_impl.cpp", test["support_sources"]) + + def test_icu_tests_retain_embedded_data_initializer(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + by_name = {test["name"]: test for test in tests} + for name in ("mongo/util/icu_test", "mongo/db/query/collation/collator_factory_icu_test", + "mongo/db/query/collation/collator_interface_icu_test"): + with self.subTest(name=name): + self.assertIn("mongo/util/icu_init.cpp", by_name[name]["support_sources"]) + + def test_sharded_pipeline_is_only_in_full_build(self): + common = "mongo/db/pipeline/pipeline.cpp" + sharded = {"mongo/db/pipeline/pipeline_sharded.cpp", + "mongo/db/pipeline/cluster_aggregation_planner.cpp"} + self.assertIn(common, self.full_pipeline_sources) + self.assertTrue(sharded <= self.full_pipeline_sources) + self.assertIn(common, self.sources) + self.assertFalse(sharded & self.sources) + + tests, support = select_minimal_tests(self.targets, self.selected) + standalone_sources = self.sources | support | { + source for test in tests for source in test["sources"]} + self.assertFalse(sharded & standalone_sources) + self.assertNotIn("mongo/db/pipeline/cluster_aggregation_planner_standalone.cpp", + standalone_sources) + + def test_tests_cover_server_but_not_data_substrate(self): + tests, support = select_minimal_tests(self.targets, self.selected) + required = { + "mongo/db/standalone_runtime_test", + "mongo/db/query/get_executor_test", + "mongo/db/commands/plan_cache_commands_test", + "mongo/db/commands/index_filter_commands_test", + "mongo/db/storage/ephemeral_for_test/storage_ephemeral_for_test_record_store_test", + } + self.assertFalse(required - {test["name"] for test in tests}) + for source in support | {src for test in tests for src in test["sources"]}: + self.assertNotIn("/data_substrate/", source) + + def test_replication_suites_are_excluded(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + for test in tests: + self.assertFalse(test["name"].startswith("mongo/db/repl/"), test["name"]) + + def test_distributed_suites_are_excluded_without_dropping_shared_coverage(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + names = {test["name"] for test in tests} + excluded = set(EXCLUDED_DISTRIBUTED_TEST_TARGETS) + self.assertTrue(excluded <= self.targets.keys()) + self.assertFalse(excluded & names) + for name in names: + self.assertFalse(name.startswith(("mongo/db/repl/", "mongo/db/s/", "mongo/s/")), name) + # Keep local request lifecycle, wire-format, metadata, and catalog coverage. + self.assertTrue({ + "mongo/db/operation_context_test", + "mongo/db/standalone_runtime_test", + "mongo/db/logical_time_test", + "mongo/db/logical_session_cache_test", + "mongo/db/catalog/collection_options_test", + "mongo/db/catalog/uuid_catalog_test", + "mongo/rpc/client_metadata_test", + "mongo/rpc/rpc_metadata_test", + "mongo/client/read_preference_test", + } <= names) + + def test_auth_tests_retain_name_registered_implementations(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + auth = next(test for test in tests + if test["name"] == "mongo/db/auth/authorization_session_test") + self.assertIn("mongo/db/auth/authorization_manager_global.cpp", auth["support_sources"]) + self.assertIn("mongo/db/pipeline/document_source_lookup.cpp", auth["support_sources"]) + self.assertIn("mongo/db/pipeline/document_source_facet.cpp", auth["support_sources"]) + + def test_query_tests_retain_authorization_factories(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + required = { + "mongo/db/auth/authorization_manager_global.cpp", + "mongo/db/auth/authorization_manager_impl.cpp", + "mongo/db/auth/authorization_session_impl.cpp", + } + for name in ("mongo/db/query/get_executor_test", + "mongo/db/commands/plan_cache_commands_test", + "mongo/db/commands/index_filter_commands_test", + "mongo/db/operation_context_test"): + with self.subTest(name=name): + test = next(test for test in tests if test["name"] == name) + self.assertFalse(required - set(test["support_sources"])) + + def test_test_only_engine_and_parser_registrations(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + by_name = {test["name"]: test for test in tests} + engine = "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_init.cpp" + self.assertNotIn(engine, self.sources) + self.assertIn(engine, by_name["mongo/db/exec/sort_test"]["support_sources"]) + for name in ("mongo/db/exec/sort_test", "mongo/db/exec/queued_data_stage_test", + "mongo/db/concurrency/lock_manager_test"): + self.assertIn("mongo/db/catalog/database_holder_impl.cpp", + by_name[name]["support_sources"]) + self.assertIn("mongo/db/catalog/collection_impl.cpp", by_name[name]["support_sources"]) + self.assertIn("mongo/db/pipeline/document_source_internal_split_pipeline.cpp", + by_name["mongo/db/pipeline/document_source_facet_test"]["support_sources"]) + self.assertIn("mongo/db/pipeline/granularity_rounder_powers_of_two.cpp", + by_name["mongo/db/pipeline/granularity_rounder_test"]["support_sources"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/how-to-compile.md b/docs/how-to-compile.md index 139b1d473b..4013257d40 100644 --- a/docs/how-to-compile.md +++ b/docs/how-to-compile.md @@ -1,73 +1,294 @@ -# How To Compile EloqDoc-RocksDB +# Build EloqDoc -```bash -git clone --recurse-submodules https://github.com/eloqdata/eloqdoc -``` +EloqDoc supports two build systems in the same source tree: + +- The top-level CMake/Ninja build is the recommended server-only build. It builds the + MongoDB-compatible single-node server, the Eloq storage-engine integration, and the in-tree Data + Substrate in one graph. It uses Python 3 and does not invoke SCons. +- The historical MongoDB 4.0 SCons build remains supported for existing development and packaging + workflows. It uses Python 2.7 and builds the Eloq core libraries separately with CMake. -## 0. Install dependencies +Use separate build directories and installation prefixes if both builds are used from one checkout. -Assuming Ubuntu 24.04 is your development environment, run the following script to install dependencies. +The top-level CMake server uses the existing EloqDoc `dbmain.cpp` / `db.cpp` startup and +`ServiceEntryPointMongod` request path. It retains authentication and authorization, TTL and +background indexes, background maintenance, session cleanup, diagnostics, and the MozJS +JavaScript engine. It does not use the embedded application runtime. + +Both builds use the vendored ICU 57.1 implementation and the same embedded collation data. +CMake does not select the host's ICU: its version is recorded in index collation metadata, +so switching to a newer system version would change compatibility with SCons-created indexes. +Existing Eloq storage-engine restrictions still apply; for example, `mapReduce` remains explicitly +unsupported even though server-side JavaScript such as `$where` is available. +The test-only `restartCatalog` command is also rejected for Eloq in both builds: its no-op MongoDB +locker cannot guarantee that other workers stop using catalog entries during a live reload. + +The build excludes `mongos`, MongoDB replica-set coordination and initial sync, WiredTiger, +the MMAPv1 and Mobile engine registrations, and distributed MongoDB aggregation. A standalone +implementation supplies the shared replication interface without starting replica-set workers; +Eloq Data Substrate continues to provide its own distributed transactions and durability. +Sharding-only pipeline methods live in `pipeline_sharded.cpp`, which SCons builds alongside the +original distributed aggregation planner. CMake omits both files; the shared `pipeline.cpp` +handles local aggregation without distributed-planner rejection stubs. +Some `repl/`, `s/`, and MMAPv1 sources remain because MongoDB's +shared command/catalog code uses their wire types, interfaces, and legacy B-tree helpers; the +distributed runtimes and alternate storage backends themselves are excluded and audited during +configuration. + +## 1. Get the source and dependencies ```bash -mkdir -p /tmp/eloqdoc-deps +git clone --recurse-submodules https://github.com/eloqdata/eloqdoc +cd eloqdoc bash scripts/install_dependency_ubuntu2404.sh /tmp/eloqdoc-deps ``` -If you are using another Linux distribution, follow the steps in `install_dependency_ubuntu2404.sh` to install dependencies manually. +The dependency script supports Ubuntu 24.04. It installs CMake, Ninja, Python 3, ICU, Python 2.7 +with the historical SCons requirements, and the Data Substrate dependency prefix. Set +`ELOQ_INSTALL_LEGACY_SCONS=0` when preparing a CMake-only environment. On another Linux +distribution, install equivalent packages and build the dependencies described by +`src/mongo/db/modules/eloq/data_substrate/scripts/third_party/install-ubuntu2404.sh`. -EloqDoc still uses the MongoDB 4.0 `scons` build, which requires Python 2.7 and its build-time packages. Ubuntu 24.04 does not provide Python 2.7 through the default apt repositories, so the dependency script installs Python 2.7.18 with `pyenv`. If you use a custom image or install dependencies manually, make sure `python2` and `pip2` are available on `PATH` before building. +By default the Data Substrate dependencies are installed at: -```bash -python2 --version -pip2 --version +```text +src/mongo/db/modules/eloq/data_substrate/third_party/install ``` -## 1. Compile EloqDoc-RocksDB +Set `ELOQ_THIRD_PARTY_PREFIX` before running the installer and either build system to use a different +absolute prefix. + +Use a private dependency prefix, not `/usr`: the SCons flags and CI helper add +`-isystem /include`. Adding `/usr/include` this way can break GCC's +`#include_next` lookup in the C++ standard-library headers. -There are two components to compile: EloqDoc and core libraries. EloqDoc is compiled with `scons`, and the core libraries are compiled with `cmake`. +## 2. Build with top-level CMake and Ninja -### 1.1 Define an installation path +From the repository root: ```bash export INSTALL_PREFIX=/absolute/path/to/install +export ELOQ_THIRD_PARTY_PREFIX="$PWD/src/mongo/db/modules/eloq/data_substrate/third_party/install" + +cmake -S . -B build/cmake -G Ninja \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_PREFIX" \ + -DELOQ_THIRD_PARTY_PREFIX="$ELOQ_THIRD_PARTY_PREFIX" \ + -DWITH_DATA_STORE=ELOQDSS_ROCKSDB \ + -DWITH_LOG_STATE=ROCKSDB + +cmake --build build/cmake -j8 +cmake --install build/cmake ``` -The `INSTALL_PREFIX` must be an absolute path. Binaries and libraries will be installed under this directory. +To select GCC 15, add `-DCMAKE_C_COMPILER=gcc-15 -DCMAKE_CXX_COMPILER=g++-15` to +the configure command and use a separate build directory such as `build/cmake-gcc15`. +Do not change compilers inside an existing build directory; keep the GCC 13, GCC 15, +and SCons build trees separate. Selecting GCC 15 keeps the MongoDB sources in C++17 mode. -### 1.2 Compile core libraries +The server is installed as `$INSTALL_PREFIX/bin/eloqdoc`. Eloq is the default and only registered +storage engine; explicitly selecting another engine is rejected. MongoDB replica-set, shard-server, +and config-server startup options are rejected. Normal server options, including `--auth`, +`--config`, and `--fork`, use the existing server parser and startup handling. -Run the following commands from the repository root: +The current Linux profile builds MongoDB without client-facing TLS, matching the historical SCons +configuration documented below. Data Substrate's separate TLS dependencies are unaffected. -```bash -cmake -S src/mongo/db/modules/eloq \ - -B src/mongo/db/modules/eloq/build \ - -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \ - -DWITH_DATA_STORE=ELOQDSS_ROCKSDB -cmake --build src/mongo/db/modules/eloq/build -j8 -cmake --install src/mongo/db/modules/eloq/build -``` +IDL generation uses per-output depfiles, including transitive imports, so editing one IDL file +regenerates only its dependent outputs. The historical SCons IDL compiler is unchanged. -### 1.3 Compile EloqDoc-RocksDB +Common data-store choices are: -The `scons` build tool depends on Python 2.7. Switch to a Python 2.7 environment before running `scons`. The `install_dependency_ubuntu2404.sh` script installs Python 2.7.18 and the required packages. +| `WITH_DATA_STORE` | Matching `WITH_LOG_STATE` | +|---|---| +| `ROCKSDB` or `ELOQDSS_ROCKSDB` | `ROCKSDB` | +| `ELOQDSS_ROCKSDB_CLOUD_S3` | `ROCKSDB_CLOUD_S3` | +| `ELOQDSS_ROCKSDB_CLOUD_GCS` | `ROCKSDB_CLOUD_GCS` | +| `ELOQDSS_ELOQSTORE` | `ROCKSDB`, `ROCKSDB_CLOUD_S3`, or `ROCKSDB_CLOUD_GCS` | -```bash -pyenv global 2.7.18 -``` +To make an AddressSanitizer build, add `-DELOQDOC_ENABLE_ASAN=ON`. -Compile EloqDoc from the repository root. +## 3. Build with SCons + +The SCons path first builds and installs the Eloq core libraries, then runs MongoDB's historical +build. Use an installation prefix separate from the top-level CMake build: ```bash +export SCONS_INSTALL_PREFIX=/absolute/path/to/scons-install +export ELOQ_THIRD_PARTY_PREFIX="$PWD/src/mongo/db/modules/eloq/data_substrate/third_party/install" + +cmake -S src/mongo/db/modules/eloq \ + -B build/eloq-scons-deps \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX="$SCONS_INSTALL_PREFIX" \ + -DELOQ_THIRD_PARTY_PREFIX="$ELOQ_THIRD_PARTY_PREFIX" \ + -DWITH_DATA_STORE=ELOQDSS_ROCKSDB \ + -DWITH_LOG_STATE=ROCKSDB +cmake --build build/eloq-scons-deps -j8 +cmake --install build/eloq-scons-deps + +pyenv global 2.7.18 env WITH_DATA_STORE=ELOQDSS_ROCKSDB \ -python scripts/buildscripts/scons.py \ + WITH_LOG_STATE=ROCKSDB \ + ELOQ_THIRD_PARTY_PREFIX="$ELOQ_THIRD_PARTY_PREFIX" \ +python2 scripts/buildscripts/scons.py \ MONGO_VERSION=4.0.3 \ VARIANT_DIR=RelWithDebInfo \ - CXXFLAGS="-Wno-nonnull -Wno-class-memaccess -Wno-interference-size -Wno-redundant-move" \ - --build-dir=#build \ - --prefix=$INSTALL_PREFIX \ + CXXFLAGS="-isystem $ELOQ_THIRD_PARTY_PREFIX/include -Wno-nonnull -Wno-class-memaccess -Wno-interference-size -Wno-redundant-move" \ + CPPDEFINES="ELOQ_MODULE_ENABLED EXT_TX_PROC_ENABLED" \ + LIBPATH="$ELOQ_THIRD_PARTY_PREFIX/lib $ELOQ_THIRD_PARTY_PREFIX/lib64" \ + --build-dir=#build/scons \ + --prefix="$SCONS_INSTALL_PREFIX" \ + --allocator=system \ + --link-model=dynamic \ + --install-mode=hygienic \ --disable-warnings-as-errors \ -j8 \ install-core ``` -All executable files will be installed to `$INSTALL_PREFIX/bin`, and all libraries will be installed to `$INSTALL_PREFIX/lib`. +This path retains the existing SCons target definitions and does not define the CMake-only +`ELOQDOC_STANDALONE` compile definition. + +### SCons native-test validation profile + +The historical `all`/`install-tests` aliases include unsupported engines, embedded executables, +and upstream tests with obsolete fork interfaces. For EloqDoc validation, filter the generated +unit-test manifest explicitly (Python 3 is used only for this selector; SCons/resmoke use Python 2): + +```bash +python3 scripts/buildscripts/select_eloq_unit_tests.py \ + --manifest build/scons/unittests.txt \ + --output build/scons/eloq-unittests.txt \ + --report build/scons/eloq-unittest-selection.json \ + --suite build/scons/eloq-unittests.yml +``` + +Build the binary paths listed in `eloq-unittests.txt` as explicit SCons targets, using the same +compiler/options/build directory as the server command above, instead of `all` or `install-tests`. +For example, in Bash, `mapfile -t test_targets < build/scons/eloq-unittests.txt` supplies the +targets as `"${test_targets[@]}"`. Then run: + +```bash +python2 scripts/buildscripts/resmoke.py \ + --suites=build/scons/eloq-unittests.yml --jobs=4 --continueOnFailure +``` + +The selection report records each skipped target and its reason. MongoDB replication tests +(`mongo/db/repl/`), MMAPv1, WiredTiger, embedded tests, and individually listed obsolete +interfaces/fixtures are excluded; legacy `dbtest` is not part of this profile. Missing retained +binaries are **not** automatically skipped. Keep the build/install +library directories on `LD_LIBRARY_PATH` for dynamic SCons tests, including test-helper libraries +that are not installed by `install-core`. Limit build/test parallelism to eight; on memory-limited +WSL hosts, use two compiler jobs and one linker job (`-j2 --jlink=1`). CPU affinity (`taskset -c 0-7` +on a host with these CPUs available) also bounds internal worker threads. + +The `ephemeralForTest` fixture implements the fork's namespace-keyed catalog metadata, +enumeration, collection lookup/version, and cursor ownership interfaces. Its index cursors also +return the documents required by Eloq's fast `_id` lookup. This is a test-only in-memory model, +not a test of distributed storage. Native regression tests cover catalog commit/rollback, +duplicate namespaces, feature documents, and index lookup with a prefetched document. + +All suites under `mongo/db/repl/` are skipped, including replication-process, consistency-marker, +drop-pending reaper, replica-set coordinator, and initial-sync tests. This matches CMake's existing +replication-test exclusion; it changes validation selection, not the historical SCons targets or +the shared interfaces used by the standalone server. + +Outside that directory, confirmed fixtures requiring MongoDB's replication oplog, local-shard +locking, capped conversion, or removed local-catalog rename/ident-repair APIs are excluded by exact +target in `scripts/buildscripts/select_eloq_unit_tests.py`. These are unsupported/obsolete fixtures, +not WiredTiger failures and not repaired tests. There is no blanket catalog or sharding-directory +exclusion: collection metadata, KV collection catalog, repair observer, and free-monitoring tests +remain selected. New or unexplained failures outside the documented exclusions remain failures. +Do not remove production `MONGO_UNREACHABLE` guards or return empty mock catalogs to make obsolete +fixtures pass. + +The nesting-depth integration test runs with either engine: it expects Eloq update failures at +command level (command-level transaction rollback), and upstream-engine failures in `writeErrors`. +Both branches check `Overflow` and verify that the rejected update leaves the document unchanged. + +## 4. Build and run CMake tests + +Enable the tests when configuring the same build directory: + +```bash +cmake -S . -B build/cmake -G Ninja \ + -DELOQDOC_BUILD_TESTS=ON + +cmake --build build/cmake --target eloqdoc-tests -j8 +ctest --test-dir build/cmake --output-on-failure -j8 +``` + +The generated test graph contains the C++ unit and integration-test declarations belonging to the +retained single-node MongoDB modules, plus the Eloq storage-integration tests declared by the Eloq +Mongo module. Tests for `mongos`, replication runtimes, and excluded storage engines are omitted. +The CMake-only exclusions also cover distributed-only suites outside those directories: +replica-set clients/monitors, cluster-time signing keys/proofs, remote operation-time tracking, +and config-server/replication RPC metadata. Shared wire-format and local catalog/session tests +remain enabled. `operation_context_test` covers tracker reuse during context recycling (reuse +when exclusively owned; preserve the old tracker and replace it when shared), without enabling +the distributed-execution suites. These exclusions do not change SCons target definitions or +its validation selector. +The in-memory `ephemeralForTest` engine is linked only into unit-test fixtures that need a real +MongoDB storage engine and locker; it is not registered in the CMake server. +Data Substrate and data-store unit tests are also deliberately disabled: those components keep and +run their tests in their own repositories. + +MongoDB integration tests require an already-running EloqDoc fixture, so their binaries are built +but their server-dependent cases are not registered with CTest by default. Every integration +binary has an always-registered `startup-options` check which runs `--help` without a server; +this catches missing option-parser initializers even in normal CI. To register the full cases, +configure with: + +```bash +-DELOQDOC_REGISTER_INTEGRATION_TESTS=ON \ +-DELOQDOC_TEST_CONNECTION_STRING=localhost:27017 +``` + +Then run `ctest --test-dir build/cmake --label-regex integration --output-on-failure` after starting +the fixture. + +The existing `eloq_basic` and `eloq_core` JavaScript suites can also run against the CMake +server using an EloqDoc shell. Put the shell directory on `PATH` as well as passing resmoke's +`--mongo` option, because some tests spawn `eloqdoc-cli` by name. `set_param1.js` always runs +its generic parameter checks and runs its oplog-fetcher checks only when the server advertises +replication commands. This preserves those assertions for SCons without restoring replication +to the CMake server or excluding the whole test. + +The self-contained server smoke test requires `ELOQDSS_ROCKSDB` or `ROCKSDB`. It starts an +authenticated server with a temporary local store, two Substrate worker cores, and WAL disabled. +EloqDoc's catalog requires a data-store handler even for a disposable fixture. The test checks +CRUD, ICU 57.1 collation metadata and lookup, local aggregation, JavaScript, +background/TTL indexes, sessions and transaction commit/abort, +adaptive ingress reactors and coroutine request scheduling, unsupported-topology rejection, +maintenance-mode command dispatch, getMore term validation, fsync, diagnostics, and clean shutdown +while a long-running command is active. It enables test-only commands to +check view-catalog invalidation and safe rejection of online catalog restart; compact, touch, and +repairDatabase must still report Eloq's existing storage-engine limitations. Install the test-only +client and run: + +```bash +python3 -m pip install --target build/cmake/python 'pymongo==4.8.0' +PYTHONPATH="$PWD/build/cmake/python" \ + cmake --build build/cmake --target eloqdoc-server-smoke -j8 +``` + +The fixture is restricted to at most eight CPUs and retains its logs and data in the temporary +directory printed at startup. It tests Mongo-facing behavior, not Data Substrate/data-store +persistence or recovery. Use `--option=value` for server options with values, as in the existing +SCons launch scripts: the gflags pass can reorder separate argument values. + +## 5. Inspect the selected MongoDB graph + +The source list is derived statically from MongoDB's fine-grained `SConscript` library metadata; +SCons itself is never imported or executed. This keeps the CMake graph synchronized with the +existing module declarations while allowing explicit single-node substitutions. + +```bash +cmake --build build/cmake --target eloqdoc-source-manifest +``` + +For source/compiler validation on a machine without the Data Substrate dependency prefix, configure +with `-DELOQDOC_BUILD_DATA_SUBSTRATE=OFF`. Generated-source and individual Mongo object targets can +then be built, but the final executable is deliberately not linkable in that mode. diff --git a/scripts/buildscripts/select_eloq_unit_tests.py b/scripts/buildscripts/select_eloq_unit_tests.py new file mode 100644 index 0000000000..3b18010c4f --- /dev/null +++ b/scripts/buildscripts/select_eloq_unit_tests.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Select SCons native tests for EloqDoc, with an explicit, auditable skip report. + +This filters the SCons unit-test manifest, not the SCons target definitions. Unknown or +missing binaries are retained: a build failure must not silently turn into a test skip. +Only confirmed unsupported engines, runtimes, and legacy fixtures are excluded. +""" + +import argparse +import json +from pathlib import Path + + +EXCLUDED_PREFIXES = { + "mongo/db/repl/": "MongoDB replication tests are outside EloqDoc validation", + "mongo/db/storage/mmap_v1/": "MMAPv1 is not an EloqDoc storage engine", + "mongo/db/storage/wiredtiger/": "WiredTiger is not an EloqDoc storage engine", + "mongo/embedded/": "embedded runtime is not the EloqDoc server", +} + +# These upstream test targets still use interfaces removed by the Eloq fork. Do not +# exclude their whole directories: the other query, pipeline, and catalog tests matter. +OBSOLETE_TARGETS = { + "mongo/db/catalog/rename_collection_test": "old Collection ownership interface", + "mongo/db/commands/mr_test": "old Collection ownership interface", + "mongo/db/pipeline/document_source_test": "change-stream test uses old Collection ownership", + "mongo/db/sessions_test": "old Session constructor", + "mongo/db/storage/ephemeral_for_test/storage_ephemeral_for_test_engine_test": + "KV harness uses removed catalog namespace interface", + "mongo/db/storage/kv/kv_database_catalog_entry_test": + "removed catalog namespace interface", + "mongo/db/storage/mobile/storage_mobile_kv_engine_test": + "KV harness uses removed catalog namespace interface", + "mongo/db/views/views_test": "removed catalog namespace interface", + "mongo/s/query/cluster_find_test": "old pooled query ownership interface", + "mongo/s/sharding_routing_table_test": "old query/namespace test interface", +} + +# These fixtures create MongoDB's replication oplog during setup. EloqDoc does not +# implement that oplog; changing production catalog semantics to run them would be +# incorrect. Outside the excluded repl/ directory, keep exact targets rather than +# excluding all of s/ or catalog/: those directories retain applicable coverage. +UNSUPPORTED_RUNTIME_FIXTURES = dict.fromkeys(( + "mongo/db/catalog/database_test", + "mongo/db/catalog/drop_database_test", + "mongo/db/keys_collection_manager_sharding_test", + "mongo/db/logical_clock_test", + "mongo/db/logical_time_validator_test", + "mongo/db/op_observer_impl_test", + "mongo/db/ops/write_ops_retryability_test", + "mongo/db/s/balancer_test", + "mongo/db/s/collection_sharding_runtime_test", + "mongo/db/s/config_server_op_observer_test", + "mongo/db/s/session_catalog_migration_destination_test", + "mongo/db/s/session_catalog_migration_source_test", + "mongo/db/s/shard_server_test", + "mongo/db/s/sharding_catalog_manager_test", + "mongo/db/transaction_history_iterator_test", + "mongo/s/catalog/replset_dist_lock_manager_test", +), "fixture requires MongoDB's replication oplog") +UNSUPPORTED_RUNTIME_FIXTURES.update({ + "mongo/db/catalog/capped_utils_test": + "capped collection conversion is explicitly unsupported by EloqDoc", + "mongo/db/catalog/create_collection_test": + "fixture uses removed local-catalog renameCollection API", + "mongo/db/storage/kv/kv_storage_engine_test": + "fixture uses removed local-catalog ident reconciliation/repair APIs", + "mongo/s/client/shard_local_test": + "MongoDB local-shard fixture assumes MongoDB write-unit/lock semantics", +}) + + +def target_name(binary): + parts = Path(binary).parts + # The manifest can contain absolute paths or build-directory-relative paths. + try: + return "/".join(parts[parts.index("mongo"):]) + except ValueError: + return binary + + +def skip_reason(binary): + name = target_name(binary) + for prefix, reason in EXCLUDED_PREFIXES.items(): + if name.startswith(prefix): + return reason + return OBSOLETE_TARGETS.get(name) or UNSUPPORTED_RUNTIME_FIXTURES.get(name) + + +def select_tests(binaries): + selected, skipped = [], [] + for binary in binaries: + reason = skip_reason(binary) + if reason: + skipped.append({"test": binary, "reason": reason}) + else: + selected.append(binary) + return selected, skipped + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--report", required=True, type=Path) + parser.add_argument("--suite", type=Path, help="also write a resmoke suite (JSON/YAML)") + args = parser.parse_args() + binaries = [line.strip() for line in args.manifest.read_text().splitlines() if line.strip()] + selected, skipped = select_tests(binaries) + args.output.write_text("".join(binary + "\n" for binary in selected)) + args.report.write_text(json.dumps({"selected": selected, "skipped": skipped}, indent=2) + "\n") + if args.suite: + args.suite.write_text(json.dumps({ + "test_kind": "cpp_unit_test", + "selector": {"root": str(args.output.resolve())}, + "executor": {"config": {}}, + }, indent=2) + "\n") + print("Selected %d tests; skipped %d (reasons: %s)" % + (len(selected), len(skipped), args.report)) + + +if __name__ == "__main__": + main() diff --git a/scripts/install_dependency_ubuntu2404.sh b/scripts/install_dependency_ubuntu2404.sh index 4c68c1975a..77ea7f2c3c 100755 --- a/scripts/install_dependency_ubuntu2404.sh +++ b/scripts/install_dependency_ubuntu2404.sh @@ -7,11 +7,13 @@ set -euo pipefail # scripts/install_dependency_ubuntu2404.sh [TEMP_DIR] [--skip_eloq_common] # # TEMP_DIR controls temporary downloads and build artifacts. It defaults to -# /tmp/eloqdoc-deps. The legacy --skip_eloq_common option installs only the -# Python 2.7 build environment. +# /tmp/eloqdoc-deps. The legacy --skip_eloq_common option skips the CMake and +# Data Substrate dependencies. # # Set ELOQ_SKIP_THIRD_PARTY=1 to skip the Data Substrate third-party workspace # build. CI uses that mode when the third-party prefix is restored from cache. +# Python 2 and the SCons requirements are installed by default so the historical build keeps +# working. Set ELOQ_INSTALL_LEGACY_SCONS=0 for a smaller CMake-only environment. SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) REPO_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) @@ -132,7 +134,7 @@ install_python2() { fi if ! command -v python2 >/dev/null 2>&1; then - log_info "Installing Python 2.7.18 with pyenv" + log_info "Installing Python 2.7.18 with pyenv for the legacy SCons workflow" apt_install \ build-essential zlib1g-dev libbz2-dev liblzma-dev libreadline-dev \ libsqlite3-dev libffi-dev libssl-dev libncurses5-dev \ @@ -202,13 +204,13 @@ configure_timezone if [ "${SKIP_ELOQ_COMMON}" = false ]; then log_info "Installing EloqDoc system build packages" system_packages=( - sudo wget curl apt-utils python3 python3-dev python3-pip python3-venv + sudo wget curl apt-utils python3 python3-dev python3-pip python3-venv python3-yaml libcurl4-openssl-dev build-essential libncurses5-dev libncursesw5-dev gnutls-dev bison zlib1g-dev ccache rsync cmake ninja-build libuv1-dev git g++ gcc make openssh-client libssl-dev libgflags-dev libleveldb-dev libsnappy-dev openssl libbz2-dev liblz4-dev libzstd-dev libboost-context-dev ca-certificates libc-ares-dev libc-ares2 m4 - pkg-config tar xz-utils libreadline-dev libsqlite3-dev ncurses-dev + pkg-config tar xz-utils libreadline-dev libsqlite3-dev ncurses-dev libicu-dev tk-dev libffi-dev liblzma-dev patchelf libprotobuf-dev protobuf-compiler libjsoncpp-dev unzip ) @@ -230,6 +232,10 @@ else log_info "Skipping EloqDoc system and Data Substrate third-party dependencies" fi -install_python2 +if [ "${ELOQ_INSTALL_LEGACY_SCONS:-1}" = "1" ]; then + install_python2 +else + log_info "Skipping the Python 2/SCons environment (ELOQ_INSTALL_LEGACY_SCONS=0)" +fi log_info "EloqDoc dependency installation completed successfully" diff --git a/src/mongo/base/SConscript b/src/mongo/base/SConscript index 27c0bbf756..b0a47a68e6 100644 --- a/src/mongo/base/SConscript +++ b/src/mongo/base/SConscript @@ -40,6 +40,8 @@ env.CppUnitTest('base_test', 'data_type_validated_test.cpp', 'data_view_test.cpp', 'encoded_value_storage_test.cpp', + 'error_codes_test.cpp', + 'error_codes_test_helper.cpp', 'initializer_dependency_graph_test.cpp', 'initializer_test.cpp', 'owned_pointer_map_test.cpp', diff --git a/src/mongo/base/error_codes.tpl.h b/src/mongo/base/error_codes.tpl.h index 472c6d72f7..f536906e8a 100644 --- a/src/mongo/base/error_codes.tpl.h +++ b/src/mongo/base/error_codes.tpl.h @@ -121,11 +121,13 @@ inline bool ErrorCodes::isA(Error code) { */ namespace error_details { +// These definitions live in a header, including the explicit specializations below. +// Inline variables keep a single definition across translation units. template -constexpr bool isNamedCode = false; +inline constexpr bool isNamedCode = false; //#for $ec in $codes template <> -constexpr bool isNamedCode = true; +inline constexpr bool isNamedCode = true; //#end for MONGO_COMPILER_NORETURN void throwExceptionForStatus(const Status& status); diff --git a/src/mongo/base/error_codes_test.cpp b/src/mongo/base/error_codes_test.cpp new file mode 100644 index 0000000000..c2298de16b --- /dev/null +++ b/src/mongo/base/error_codes_test.cpp @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2026 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/base/error_codes.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace error_codes_test { + +const bool* namedCodeAddressInOtherTranslationUnit(); +const bool* unnamedCodeAddressInOtherTranslationUnit(); + +namespace { + +TEST(ErrorCodes, NamedCodeVariableTemplatesHaveOneDefinition) { + static_assert(error_details::isNamedCode, "BadValue is named"); + static_assert(!error_details::isNamedCode<-1>, "-1 is not named"); + // Taking addresses verifies the header definitions are shared across translation units, + // even on compilers that otherwise accept non-inline constexpr specializations. + ASSERT_EQ(&error_details::isNamedCode, + namedCodeAddressInOtherTranslationUnit()); + ASSERT_EQ(&error_details::isNamedCode<-1>, unnamedCodeAddressInOtherTranslationUnit()); +} + +} // namespace +} // namespace error_codes_test +} // namespace mongo diff --git a/src/mongo/base/error_codes_test_helper.cpp b/src/mongo/base/error_codes_test_helper.cpp new file mode 100644 index 0000000000..a7e4c46eaa --- /dev/null +++ b/src/mongo/base/error_codes_test_helper.cpp @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2026 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/base/error_codes.h" + +namespace mongo { +namespace error_codes_test { + +const bool* namedCodeAddressInOtherTranslationUnit() { + return &error_details::isNamedCode; +} + +const bool* unnamedCodeAddressInOtherTranslationUnit() { + return &error_details::isNamedCode<-1>; +} + +} // namespace error_codes_test +} // namespace mongo diff --git a/src/mongo/bson/simple_bsonobj_comparator.h b/src/mongo/bson/simple_bsonobj_comparator.h index 59a61e27dd..738adeda59 100644 --- a/src/mongo/bson/simple_bsonobj_comparator.h +++ b/src/mongo/bson/simple_bsonobj_comparator.h @@ -73,7 +73,7 @@ class SimpleBSONObjComparator final : public BSONObj::ComparatorInterface { */ class EqualTo { public: - explicit EqualTo() = default; + EqualTo() = default; bool operator()(const BSONObj& lhs, const BSONObj& rhs) const { return kInstance.compare(lhs, rhs) == 0; @@ -86,7 +86,7 @@ class SimpleBSONObjComparator final : public BSONObj::ComparatorInterface { */ class Hasher { public: - explicit Hasher() = default; + Hasher() = default; size_t operator()(const BSONObj& obj) const { return kInstance.hash(obj); diff --git a/src/mongo/bson/simple_bsonobj_comparator_test.cpp b/src/mongo/bson/simple_bsonobj_comparator_test.cpp index 40d536f0bd..e5e50c513d 100644 --- a/src/mongo/bson/simple_bsonobj_comparator_test.cpp +++ b/src/mongo/bson/simple_bsonobj_comparator_test.cpp @@ -58,6 +58,17 @@ void assertInsertFails(Set& set, BSONObj obj) { << "object was inserted successfully, but should have failed: " << obj.jsonString(); } +TEST(SimpleBSONObjContainerTest, UnorderedFunctorsAreCopyListInitializable) { + // Some standard-library hash tables initialize these stateless functors from an empty list. + SimpleBSONObjComparator::EqualTo equalTo = {}; + SimpleBSONObjComparator::Hasher hasher = {}; + const auto obj = BSON("x" << 1); + const auto same = BSON("x" << 1); + ASSERT_TRUE(equalTo(obj, same)); + ASSERT_FALSE(equalTo(obj, BSON("x" << 2))); + ASSERT_EQ(hasher(obj), hasher(same)); +} + TEST(SimpleBSONObjContainerTest, SetIsDefaultConstructible) { SimpleBSONObjSet set; assertInsertSucceeds(set, BSON("x" << 1)); diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index 9f92e861f5..fe88fe1953 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -1612,6 +1612,7 @@ env.CppUnitTest( ], LIBDEPS=[ 'logical_time', + 'service_context_test_fixture', ], ) diff --git a/src/mongo/db/auth/authorization_manager_test.cpp b/src/mongo/db/auth/authorization_manager_test.cpp index e0ba470d4f..186205f2ba 100644 --- a/src/mongo/db/auth/authorization_manager_test.cpp +++ b/src/mongo/db/auth/authorization_manager_test.cpp @@ -49,6 +49,7 @@ #include "mongo/db/operation_context.h" #include "mongo/db/service_context_test_fixture.h" #include "mongo/db/storage/recovery_unit_noop.h" +#include "mongo/db/storage/storage_engine.h" #include "mongo/stdx/memory.h" #include "mongo/transport/session.h" #include "mongo/transport/transport_layer_mock.h" @@ -74,6 +75,64 @@ void setX509PeerInfo(const transport::SessionHandle& session, SSLPeerInfo info) using std::vector; +// These tests exercise local authorization bookkeeping, not Eloq's cross-node cache +// notifications. Satisfy the storage-engine dependency without starting a real engine. +class AuthorizationStorageEngineMock final : public StorageEngine { +public: + void onAuthzDataChanged(OperationContext*) override {} + + RecoveryUnit* newRecoveryUnit() override { + return new RecoveryUnitNoop(); + } + + bool supportsDocLocking() const override { + return false; + } + + bool isDurable() const override { + return false; + } + + bool isEphemeral() const override { + return true; + } + + void cleanShutdown() override {} + + // No database operations should reach the engine in these mock-backed auth tests. + void listDatabases(std::vector*) const override { + MONGO_UNREACHABLE; + } + + DatabaseCatalogEntry* getDatabaseCatalogEntry(OperationContext*, StringData) override { + MONGO_UNREACHABLE; + } + + Status closeDatabase(OperationContext*, StringData) override { + MONGO_UNREACHABLE; + } + + Status dropDatabase(OperationContext*, StringData) override { + MONGO_UNREACHABLE; + } + + int flushAllFiles(OperationContext*, bool) override { + MONGO_UNREACHABLE; + } + + Status repairRecordStore(OperationContext*, const std::string&) override { + MONGO_UNREACHABLE; + } + + void setJournalListener(JournalListener*) override { + MONGO_UNREACHABLE; + } + + Timestamp getAllCommittedTimestamp() const override { + MONGO_UNREACHABLE; + } +}; + class AuthorizationManagerTest : public ServiceContextTest { public: virtual ~AuthorizationManagerTest() { @@ -82,6 +141,7 @@ class AuthorizationManagerTest : public ServiceContextTest { } AuthorizationManagerTest() { + getServiceContext()->setStorageEngine(std::make_unique()); auto localExternalState = std::make_unique(); externalState = localExternalState.get(); auto localAuthzManager = std::make_unique( diff --git a/src/mongo/db/catalog/collection_mock.h b/src/mongo/db/catalog/collection_mock.h index 2b435027c6..d13c70005c 100644 --- a/src/mongo/db/catalog/collection_mock.h +++ b/src/mongo/db/catalog/collection_mock.h @@ -62,12 +62,16 @@ class CollectionMock : virtual public Collection::Impl, Status recordStoreGoingToUpdateInPlace(OperationContext* opCtx, const RecordId& loc) { std::abort(); } + const std::string _catalogVersion; const NamespaceString _ns; public: const NamespaceString& ns() const { return _ns; } + const std::string& catalogVersion() const { + return _catalogVersion; + } bool ok() const { std::abort(); } @@ -138,6 +142,17 @@ class CollectionMock : virtual public Collection::Impl, std::abort(); } + void deleteDocument(OperationContext* opCtx, + StmtId stmtId, + const RecordId& loc, + const Snapshotted& doc, + OpDebug* opDebug, + bool fromMigrate, + bool noWarn, + Collection::StoreDeletedDoc storeDeletedDoc) { + std::abort(); + } + Status insertDocuments(OperationContext* opCtx, std::vector::const_iterator begin, std::vector::const_iterator end, diff --git a/src/mongo/db/catalog/database_holder_mock.h b/src/mongo/db/catalog/database_holder_mock.h index ab4058e441..9d0d93df06 100644 --- a/src/mongo/db/catalog/database_holder_mock.h +++ b/src/mongo/db/catalog/database_holder_mock.h @@ -51,7 +51,7 @@ class DatabaseHolderMock : public DatabaseHolder::Impl { * Retrieves an already opened database or returns nullptr. Must be called with the database * locked in at least IS-mode. */ - Database* get(OperationContext* opCtx, StringData ns) const override { + Database* get(OperationContext* opCtx, StringData ns) override { return nullptr; } diff --git a/src/mongo/db/catalog/uuid_catalog_test.cpp b/src/mongo/db/catalog/uuid_catalog_test.cpp index 3db1297667..344533d8bb 100644 --- a/src/mongo/db/catalog/uuid_catalog_test.cpp +++ b/src/mongo/db/catalog/uuid_catalog_test.cpp @@ -43,7 +43,6 @@ class UUIDCatalogTest : public unittest::Test { public: UUIDCatalogTest() : nss("testdb", "testcol"), - col(stdx::make_unique(nss)), colUUID(CollectionUUID::gen()), nextUUID(CollectionUUID::gen()), prevUUID(CollectionUUID::gen()) { @@ -57,14 +56,30 @@ class UUIDCatalogTest : public unittest::Test { ASSERT_GT(nextUUID, colUUID); // Register dummy collection in catalog. - catalog.onCreateCollection(&opCtx, &col, colUUID); + col = createCollection(nss, colUUID); + } + + Collection* createCollection(const NamespaceString& collectionNss, CollectionUUID uuid) { + auto owned = stdx::make_unique( + stdx::make_unique(collectionNss)); + auto* collection = owned.get(); + catalog.onCreateCollection(&opCtx, std::move(owned), uuid); + return collection; + } + + Collection* renameCollection(const NamespaceString& collectionNss, CollectionUUID uuid) { + auto owned = stdx::make_unique( + stdx::make_unique(collectionNss)); + auto* collection = owned.get(); + catalog.onRenameCollection(&opCtx, std::move(owned), uuid); + return collection; } protected: UUIDCatalog catalog; OperationContextNoop opCtx; NamespaceString nss; - Collection col; + Collection* col; CollectionUUID colUUID; CollectionUUID nextUUID; CollectionUUID prevUUID; @@ -73,7 +88,7 @@ class UUIDCatalogTest : public unittest::Test { namespace { TEST_F(UUIDCatalogTest, OnCreateCollection) { - ASSERT(catalog.lookupCollectionByUUID(colUUID) == &col); + ASSERT(catalog.lookupCollectionByUUID(colUUID).get() == col); } TEST_F(UUIDCatalogTest, LookupCollectionByUUID) { @@ -94,13 +109,12 @@ TEST_F(UUIDCatalogTest, LookupNSSByUUID) { TEST_F(UUIDCatalogTest, InsertAfterLookup) { auto newUUID = CollectionUUID::gen(); NamespaceString newNss(nss.db(), "newcol"); - Collection newCol(stdx::make_unique(newNss)); // Ensure that looking up non-existing UUIDs doesn't affect later registration of those UUIDs. ASSERT(catalog.lookupCollectionByUUID(newUUID) == nullptr); ASSERT(catalog.lookupNSSByUUID(newUUID) == NamespaceString()); - catalog.onCreateCollection(&opCtx, &newCol, newUUID); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(newUUID), &newCol); + auto* newCol = createCollection(newNss, newUUID); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(newUUID).get(), newCol); ASSERT_EQUALS(catalog.lookupNSSByUUID(colUUID), nss); } @@ -113,14 +127,12 @@ TEST_F(UUIDCatalogTest, OnDropCollection) { TEST_F(UUIDCatalogTest, OnRenameCollection) { auto oldUUID = CollectionUUID::gen(); NamespaceString oldNss(nss.db(), "oldcol"); - Collection oldCol(stdx::make_unique(oldNss)); - catalog.onCreateCollection(&opCtx, &oldCol, oldUUID); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(oldUUID), &oldCol); + auto* oldCol = createCollection(oldNss, oldUUID); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(oldUUID).get(), oldCol); NamespaceString newNss(nss.db(), "newcol"); - Collection newCol(stdx::make_unique(newNss)); - catalog.onRenameCollection(&opCtx, &newCol, oldUUID); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(oldUUID), &newCol); + auto* newCol = renameCollection(newNss, oldUUID); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(oldUUID).get(), newCol); } TEST_F(UUIDCatalogTest, NonExistingNextCol) { @@ -128,20 +140,17 @@ TEST_F(UUIDCatalogTest, NonExistingNextCol) { ASSERT_FALSE(catalog.next(nss.db(), nextUUID)); NamespaceString newNss("anotherdb", "newcol"); - Collection newCol(stdx::make_unique(newNss)); - catalog.onCreateCollection(&opCtx, &newCol, nextUUID); + createCollection(newNss, nextUUID); ASSERT_FALSE(catalog.next(nss.db(), colUUID)); NamespaceString prevNss(nss.db(), "prevcol"); - Collection prevCol(stdx::make_unique(prevNss)); - catalog.onCreateCollection(&opCtx, &prevCol, prevUUID); + createCollection(prevNss, prevUUID); ASSERT_FALSE(catalog.next(nss.db(), colUUID)); } TEST_F(UUIDCatalogTest, ExistingNextCol) { NamespaceString nextNss(nss.db(), "next"); - Collection nextCol(stdx::make_unique(nextNss)); - catalog.onCreateCollection(&opCtx, &nextCol, nextUUID); + createCollection(nextNss, nextUUID); auto next = catalog.next(nss.db(), colUUID); ASSERT_TRUE(next); ASSERT_EQUALS(*next, nextUUID); @@ -152,20 +161,17 @@ TEST_F(UUIDCatalogTest, NonExistingPrevCol) { ASSERT_FALSE(catalog.prev(nss.db(), prevUUID)); NamespaceString newNss("anotherdb", "newcol"); - Collection newCol(stdx::make_unique(newNss)); - catalog.onCreateCollection(&opCtx, &newCol, nextUUID); + createCollection(newNss, nextUUID); ASSERT_FALSE(catalog.prev(nss.db(), colUUID)); NamespaceString nextNss(nss.db(), "nextcol"); - Collection nextCol(stdx::make_unique(nextNss)); - catalog.onCreateCollection(&opCtx, &nextCol, nextUUID); + createCollection(nextNss, nextUUID); ASSERT_FALSE(catalog.prev(nss.db(), colUUID)); } TEST_F(UUIDCatalogTest, ExistingPrevCol) { NamespaceString prevNss(nss.db(), "prevcol"); - Collection prevCol(stdx::make_unique(prevNss)); - catalog.onCreateCollection(&opCtx, &prevCol, prevUUID); + createCollection(prevNss, prevUUID); auto prev = catalog.prev(nss.db(), colUUID); ASSERT_TRUE(prev); ASSERT_EQUALS(*prev, prevUUID); @@ -181,12 +187,10 @@ TEST_F(UUIDCatalogTest, NextPrevColOnEmptyCatalog) { TEST_F(UUIDCatalogTest, InvalidateOrdering) { NamespaceString prevNss(nss.db(), "prevcol"); - Collection prevCol(stdx::make_unique(prevNss)); - catalog.onCreateCollection(&opCtx, &prevCol, prevUUID); + createCollection(prevNss, prevUUID); NamespaceString nextNss(nss.db(), "nextcol"); - Collection nextCol(stdx::make_unique(nextNss)); - catalog.onCreateCollection(&opCtx, &nextCol, nextUUID); + createCollection(nextNss, nextUUID); catalog.onDropCollection(&opCtx, colUUID); @@ -211,36 +215,34 @@ TEST_F(UUIDCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsOldNSSIfDropped) { TEST_F(UUIDCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsNewlyCreatedNSS) { auto newUUID = CollectionUUID::gen(); NamespaceString newNss(nss.db(), "newcol"); - Collection newCol(stdx::make_unique(newNss)); // Ensure that looking up non-existing UUIDs doesn't affect later registration of those UUIDs. catalog.onCloseCatalog(&opCtx); ASSERT(catalog.lookupCollectionByUUID(newUUID) == nullptr); ASSERT(catalog.lookupNSSByUUID(newUUID) == NamespaceString()); - catalog.onCreateCollection(&opCtx, &newCol, newUUID); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(newUUID), &newCol); + auto* newCol = createCollection(newNss, newUUID); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(newUUID).get(), newCol); ASSERT_EQUALS(catalog.lookupNSSByUUID(colUUID), nss); // Ensure that collection still exists after opening the catalog again. catalog.onOpenCatalog(&opCtx); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(newUUID), &newCol); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(newUUID).get(), newCol); ASSERT_EQUALS(catalog.lookupNSSByUUID(colUUID), nss); } TEST_F(UUIDCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsFreshestNSS) { NamespaceString newNss(nss.db(), "newcol"); - Collection newCol(stdx::make_unique(newNss)); catalog.onCloseCatalog(&opCtx); catalog.onDropCollection(&opCtx, colUUID); ASSERT(catalog.lookupCollectionByUUID(colUUID) == nullptr); ASSERT_EQUALS(catalog.lookupNSSByUUID(colUUID), nss); - catalog.onCreateCollection(&opCtx, &newCol, colUUID); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(colUUID), &newCol); + auto* newCol = createCollection(newNss, colUUID); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(colUUID).get(), newCol); ASSERT_EQUALS(catalog.lookupNSSByUUID(colUUID), newNss); // Ensure that collection still exists after opening the catalog again. catalog.onOpenCatalog(&opCtx); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(colUUID), &newCol); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(colUUID).get(), newCol); ASSERT_EQUALS(catalog.lookupNSSByUUID(colUUID), newNss); } } // namespace diff --git a/src/mongo/db/catalog_raii_test.cpp b/src/mongo/db/catalog_raii_test.cpp index 4d178e9d72..07b727e8af 100644 --- a/src/mongo/db/catalog_raii_test.cpp +++ b/src/mongo/db/catalog_raii_test.cpp @@ -37,6 +37,7 @@ #include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/concurrency/lock_state.h" #include "mongo/db/service_context_test_fixture.h" +#include "mongo/db/storage/recovery_unit_noop.h" #include "mongo/unittest/unittest.h" #include "mongo/util/log.h" #include "mongo/util/time_support.h" @@ -52,7 +53,16 @@ class CatalogRAIITestFixture : public ServiceContextTest { ClientAndCtx makeClientWithLocker(const std::string& clientName) { auto client = getServiceContext()->makeClient(clientName); auto opCtx = client->makeOperationContext(); - opCtx->swapLockState(stdx::make_unique()); + // This fixture tests locking without a storage engine. Install its own state + // on both newly allocated and pooled operation contexts. + auto locker = stdx::make_unique(); + if (opCtx->lockState()) { + opCtx->swapLockState(std::move(locker)); + } else { + opCtx->setLockState(std::move(locker)); + } + opCtx->setRecoveryUnit(new RecoveryUnitNoop(), + WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); return std::make_pair(std::move(client), std::move(opCtx)); } diff --git a/src/mongo/db/commands/conn_pool_stats.cpp b/src/mongo/db/commands/conn_pool_stats.cpp index f554b55302..d3411e4617 100644 --- a/src/mongo/db/commands/conn_pool_stats.cpp +++ b/src/mongo/db/commands/conn_pool_stats.cpp @@ -114,6 +114,7 @@ class PoolStats final : public BasicCommand { } poolStatsCmd; +#ifndef ELOQDOC_STANDALONE class ShardedPoolStats final : public BasicCommand { public: ShardedPoolStats() : BasicCommand("shardConnPoolStats") {} @@ -156,6 +157,7 @@ class ShardedPoolStats final : public BasicCommand { } } shardedPoolStatsCmd; +#endif } // namespace } // namespace mongo diff --git a/src/mongo/db/commands/conn_pool_sync.cpp b/src/mongo/db/commands/conn_pool_sync.cpp index 31b9291357..7fbc91e9e6 100644 --- a/src/mongo/db/commands/conn_pool_sync.cpp +++ b/src/mongo/db/commands/conn_pool_sync.cpp @@ -63,7 +63,9 @@ class PoolFlushCmd : public BasicCommand { const std::string&, const mongo::BSONObj&, mongo::BSONObjBuilder& result) override { +#ifndef ELOQDOC_STANDALONE shardConnectionPool.flush(); +#endif globalConnPool.flush(); return true; } diff --git a/src/mongo/db/commands/fsync.cpp b/src/mongo/db/commands/fsync.cpp index 81807c7e52..13074b739f 100644 --- a/src/mongo/db/commands/fsync.cpp +++ b/src/mongo/db/commands/fsync.cpp @@ -48,7 +48,6 @@ #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db.h" #include "mongo/db/service_context.h" -#include "mongo/db/storage/mmap_v1/dur.h" #include "mongo/db/storage/storage_engine.h" #include "mongo/stdx/condition_variable.h" #include "mongo/util/assert_util.h" @@ -136,16 +135,6 @@ class FSyncCommand : public ErrmsgCommandDeprecated { if (!lock) { // the simple fsync command case - if (sync) { - // can this be GlobalRead? and if it can, it should be nongreedy. - Lock::GlobalWrite w(opCtx); - // TODO SERVER-26822: Replace MMAPv1 specific calls with ones that are storage - // engine agnostic. - getDur().commitNow(opCtx); - - // No WriteUnitOfWork needed, as this does no writes of its own. - } - // Take a global IS lock to ensure the storage engine is not shutdown Lock::GlobalLock global(opCtx, MODE_IS); StorageEngine* storageEngine = getGlobalServiceContext()->getStorageEngine(); @@ -346,16 +335,6 @@ void FSyncLockThread::run() { OperationContext& opCtx = *opCtxPtr; Lock::GlobalWrite global(&opCtx); // No WriteUnitOfWork needed - try { - // TODO SERVER-26822: Replace MMAPv1 specific calls with ones that are storage engine - // agnostic. - getDur().syncDataAndTruncateJournal(&opCtx); - } catch (const std::exception& e) { - error() << "error doing syncDataAndTruncateJournal: " << e.what(); - fsyncCmd.threadStatus = Status(ErrorCodes::CommandFailed, e.what()); - fsyncCmd.acquireFsyncLockSyncCV.notify_one(); - return; - } opCtx.lockState()->downgradeGlobalXtoSForMMAPV1(); StorageEngine* storageEngine = getGlobalServiceContext()->getStorageEngine(); diff --git a/src/mongo/db/commands/index_filter_commands_test.cpp b/src/mongo/db/commands/index_filter_commands_test.cpp index 7ee3a62e25..9a42e852e7 100644 --- a/src/mongo/db/commands/index_filter_commands_test.cpp +++ b/src/mongo/db/commands/index_filter_commands_test.cpp @@ -126,14 +126,14 @@ void addQueryShapeToPlanCache(OperationContext* opCtx, const char* projectionStr, const char* collationStr) { // Create canonical query. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projectionStr)); qr->setCollation(fromjson(collationStr)); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx, std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - std::unique_ptr cq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); QuerySolution qs; qs.cacheData.reset(new SolutionCacheData()); @@ -157,14 +157,14 @@ bool planCacheContains(OperationContext* opCtx, const char* collationStr) { // Create canonical query. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projectionStr)); qr->setCollation(fromjson(collationStr)); auto statusWithInputQuery = CanonicalQuery::canonicalize(opCtx, std::move(qr)); ASSERT_OK(statusWithInputQuery.getStatus()); - unique_ptr inputQuery = std::move(statusWithInputQuery.getValue()); + CanonicalQuery::UPtr inputQuery = std::move(statusWithInputQuery.getValue()); // Retrieve cache entries from plan cache. vector entries = planCache.getAllEntries(); @@ -177,14 +177,14 @@ bool planCacheContains(OperationContext* opCtx, // Canonicalizing query shape in cache entry to get cache key. // Alternatively, we could add key to PlanCacheEntry but that would be used in one place // only. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(entry->query); qr->setSort(entry->sort); qr->setProj(entry->projection); qr->setCollation(entry->collation); auto statusWithCurrentQuery = CanonicalQuery::canonicalize(opCtx, std::move(qr)); ASSERT_OK(statusWithCurrentQuery.getStatus()); - unique_ptr currentQuery = std::move(statusWithCurrentQuery.getValue()); + CanonicalQuery::UPtr currentQuery = std::move(statusWithCurrentQuery.getValue()); if (planCache.computeKey(*currentQuery) == planCache.computeKey(*inputQuery)) { found = true; diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp index e798ca0c9a..ddbeb70008 100644 --- a/src/mongo/db/commands/mr.cpp +++ b/src/mongo/db/commands/mr.cpp @@ -392,7 +392,9 @@ void State::dropTempCollections() { } }); // Always forget about temporary namespaces, so we don't cache lots of them +#ifndef ELOQDOC_STANDALONE ShardConnection::forgetNS(_config.tempNamespace.ns()); +#endif } if (_useIncremental && !_config.incLong.isEmpty()) { writeConflictRetry(_opCtx, "M/R dropTempCollections", _config.incLong.ns(), [this] { @@ -405,7 +407,9 @@ void State::dropTempCollections() { } }); +#ifndef ELOQDOC_STANDALONE ShardConnection::forgetNS(_config.incLong.ns()); +#endif } } @@ -1694,6 +1698,7 @@ class MapReduceCommand : public ErrmsgCommandDeprecated { /** * This class represents a map/reduce command executed on the output server of a sharded env */ +#ifndef ELOQDOC_STANDALONE class MapReduceFinishCommand : public BasicCommand { public: std::string help() const override { @@ -1893,6 +1898,7 @@ class MapReduceFinishCommand : public BasicCommand { } } mapReduceFinishCommand; +#endif } // namespace mr } // namespace mongo diff --git a/src/mongo/db/commands/plan_cache_commands_test.cpp b/src/mongo/db/commands/plan_cache_commands_test.cpp index 43d0f7d383..f80eb01614 100644 --- a/src/mongo/db/commands/plan_cache_commands_test.cpp +++ b/src/mongo/db/commands/plan_cache_commands_test.cpp @@ -136,14 +136,14 @@ TEST(PlanCacheCommandsTest, planCacheListQueryShapesOneKey) { auto opCtx = serviceContext.makeOperationContext(); // Create a canonical query - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 1}")); qr->setSort(fromjson("{a: -1}")); qr->setProj(fromjson("{_id: 0}")); qr->setCollation(fromjson("{locale: 'mock_reverse_string'}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr cq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); // Plan cache with one entry PlanCache planCache; @@ -173,11 +173,11 @@ TEST(PlanCacheCommandsTest, planCacheClearAllShapes) { auto opCtx = serviceContext.makeOperationContext(); // Create a canonical query - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 1}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr cq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); // Plan cache with one entry PlanCache planCache; @@ -235,41 +235,41 @@ TEST(PlanCacheCommandsTest, Canonicalize) { auto statusWithCQ = PlanCacheCommand::canonicalize(opCtx.get(), nss.ns(), fromjson("{query: {a: 1, b: 1}}")); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr query = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr query = std::move(statusWithCQ.getValue()); // Equivalent query should generate same key. statusWithCQ = PlanCacheCommand::canonicalize(opCtx.get(), nss.ns(), fromjson("{query: {b: 1, a: 1}}")); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr equivQuery = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr equivQuery = std::move(statusWithCQ.getValue()); ASSERT_EQUALS(planCache.computeKey(*query), planCache.computeKey(*equivQuery)); // Sort query should generate different key from unsorted query. statusWithCQ = PlanCacheCommand::canonicalize( opCtx.get(), nss.ns(), fromjson("{query: {a: 1, b: 1}, sort: {a: 1, b: 1}}")); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr sortQuery1 = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr sortQuery1 = std::move(statusWithCQ.getValue()); ASSERT_NOT_EQUALS(planCache.computeKey(*query), planCache.computeKey(*sortQuery1)); // Confirm sort arguments are properly delimited (SERVER-17158) statusWithCQ = PlanCacheCommand::canonicalize( opCtx.get(), nss.ns(), fromjson("{query: {a: 1, b: 1}, sort: {aab: 1}}")); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr sortQuery2 = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr sortQuery2 = std::move(statusWithCQ.getValue()); ASSERT_NOT_EQUALS(planCache.computeKey(*sortQuery1), planCache.computeKey(*sortQuery2)); // Changing order and/or value of predicates should not change key statusWithCQ = PlanCacheCommand::canonicalize( opCtx.get(), nss.ns(), fromjson("{query: {b: 3, a: 3}, sort: {a: 1, b: 1}}")); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr sortQuery3 = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr sortQuery3 = std::move(statusWithCQ.getValue()); ASSERT_EQUALS(planCache.computeKey(*sortQuery1), planCache.computeKey(*sortQuery3)); // Projected query should generate different key from unprojected query. statusWithCQ = PlanCacheCommand::canonicalize( opCtx.get(), nss.ns(), fromjson("{query: {a: 1, b: 1}, projection: {_id: 0, a: 1}}")); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr projectionQuery = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr projectionQuery = std::move(statusWithCQ.getValue()); ASSERT_NOT_EQUALS(planCache.computeKey(*query), planCache.computeKey(*projectionQuery)); } @@ -310,16 +310,16 @@ TEST(PlanCacheCommandsTest, planCacheClearOneKey) { auto opCtx = serviceContext.makeOperationContext(); // Create 2 canonical queries. - auto qrA = stdx::make_unique(nss); + auto qrA = ObjectPool::newObject(nss); qrA->setFilter(fromjson("{a: 1}")); auto statusWithCQA = CanonicalQuery::canonicalize(opCtx.get(), std::move(qrA)); ASSERT_OK(statusWithCQA.getStatus()); - auto qrB = stdx::make_unique(nss); + auto qrB = ObjectPool::newObject(nss); qrB->setFilter(fromjson("{b: 1}")); - unique_ptr cqA = std::move(statusWithCQA.getValue()); + CanonicalQuery::UPtr cqA = std::move(statusWithCQA.getValue()); auto statusWithCQB = CanonicalQuery::canonicalize(opCtx.get(), std::move(qrB)); ASSERT_OK(statusWithCQB.getStatus()); - unique_ptr cqB = std::move(statusWithCQB.getValue()); + CanonicalQuery::UPtr cqB = std::move(statusWithCQB.getValue()); // Create plan cache with 2 entries. PlanCache planCache; @@ -369,17 +369,17 @@ TEST(PlanCacheCommandsTest, planCacheClearOneKeyCollation) { auto opCtx = serviceContext.makeOperationContext(); // Create 2 canonical queries, one with collation. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 'foo'}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr cq = std::move(statusWithCQ.getValue()); - auto qrCollation = stdx::make_unique(nss); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); + auto qrCollation = ObjectPool::newObject(nss); qrCollation->setFilter(fromjson("{a: 'foo'}")); qrCollation->setCollation(fromjson("{locale: 'mock_reverse_string'}")); auto statusWithCQCollation = CanonicalQuery::canonicalize(opCtx.get(), std::move(qrCollation)); ASSERT_OK(statusWithCQCollation.getStatus()); - unique_ptr cqCollation = std::move(statusWithCQCollation.getValue()); + CanonicalQuery::UPtr cqCollation = std::move(statusWithCQCollation.getValue()); // Create plan cache with 2 entries. Add an index so that indexability is included in the plan // cache keys. @@ -527,11 +527,11 @@ TEST(PlanCacheCommandsTest, planCacheListPlansOnlyOneSolutionTrue) { auto opCtx = serviceContext.makeOperationContext(); // Create a canonical query - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 1}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr cq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); // Plan cache with one entry PlanCache planCache; @@ -557,11 +557,11 @@ TEST(PlanCacheCommandsTest, planCacheListPlansOnlyOneSolutionFalse) { auto opCtx = serviceContext.makeOperationContext(); // Create a canonical query - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 1}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr cq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); // Plan cache with one entry PlanCache planCache; @@ -590,17 +590,17 @@ TEST(PlanCacheCommandsTest, planCacheListPlansCollation) { auto opCtx = serviceContext.makeOperationContext(); // Create 2 canonical queries, one with collation. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 'foo'}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr cq = std::move(statusWithCQ.getValue()); - auto qrCollation = stdx::make_unique(nss); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); + auto qrCollation = ObjectPool::newObject(nss); qrCollation->setFilter(fromjson("{a: 'foo'}")); qrCollation->setCollation(fromjson("{locale: 'mock_reverse_string'}")); auto statusWithCQCollation = CanonicalQuery::canonicalize(opCtx.get(), std::move(qrCollation)); ASSERT_OK(statusWithCQCollation.getStatus()); - unique_ptr cqCollation = std::move(statusWithCQCollation.getValue()); + CanonicalQuery::UPtr cqCollation = std::move(statusWithCQCollation.getValue()); // Create plan cache with 2 entries. Add an index so that indexability is included in the plan // cache keys. Give query with collation two solutions. @@ -645,7 +645,7 @@ TEST(PlanCacheCommandsTest, planCacheListPlansTimeOfCreationIsCorrect) { auto opCtx = serviceContext.makeOperationContext(); // Create a canonical query. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 1}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); diff --git a/src/mongo/db/commands/restart_catalog_command.cpp b/src/mongo/db/commands/restart_catalog_command.cpp index d7958defdb..895912590c 100644 --- a/src/mongo/db/commands/restart_catalog_command.cpp +++ b/src/mongo/db/commands/restart_catalog_command.cpp @@ -39,6 +39,7 @@ #include "mongo/db/commands.h" #include "mongo/db/commands/test_commands_enabled.h" #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/storage/storage_options.h" #include "mongo/util/log.h" namespace mongo { @@ -87,6 +88,11 @@ class RestartCatalogCmd final : public BasicCommand { const std::string& db, const BSONObj& cmdObj, BSONObjBuilder& result) final { + // This command depends on a real global exclusive lock to protect every worker's + // catalog pointers. Eloq's no-op locker cannot provide that stop-the-world guarantee. + uassert(ErrorCodes::CommandNotSupported, + "Eloq storage engine does not support online catalog restart", + storageGlobalParams.engine != "eloq"); Lock::GlobalLock global(opCtx, MODE_X); // This command will fail without modifying the catalog if there are any databases that are diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp index 1d6ca50d71..5744da5ddb 100644 --- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp +++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp @@ -172,6 +172,7 @@ class SetFeatureCompatibilityVersionCommand : public BasicCommand { updateUniqueIndexesOnUpgrade(opCtx); +#ifndef ELOQDOC_STANDALONE // Upgrade shards before config finishes its upgrade. if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) { auto allDbs = uassertStatusOK(Grid::get(opCtx)->catalogClient()->getAllDBs( @@ -224,6 +225,7 @@ class SetFeatureCompatibilityVersionCommand : public BasicCommand { << requestedVersion))))); } +#endif FeatureCompatibilityVersion::unsetTargetUpgradeOrDowngrade(opCtx, requestedVersion); } else if (requestedVersion == FeatureCompatibilityVersionParser::kVersion36) { uassert(ErrorCodes::IllegalOperation, @@ -254,6 +256,7 @@ class SetFeatureCompatibilityVersionCommand : public BasicCommand { Lock::GlobalLock lk(opCtx, MODE_S); } +#ifndef ELOQDOC_STANDALONE // Downgrade shards before config finishes its downgrade. if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) { uassertStatusOK( @@ -303,6 +306,7 @@ class SetFeatureCompatibilityVersionCommand : public BasicCommand { Grid::get(opCtx)->catalogCache()->purgeAllDatabases(); } +#endif FeatureCompatibilityVersion::unsetTargetUpgradeOrDowngrade(opCtx, requestedVersion); } diff --git a/src/mongo/db/concurrency/d_concurrency_test.cpp b/src/mongo/db/concurrency/d_concurrency_test.cpp index 2fa499e2f0..18613c98c9 100644 --- a/src/mongo/db/concurrency/d_concurrency_test.cpp +++ b/src/mongo/db/concurrency/d_concurrency_test.cpp @@ -35,6 +35,7 @@ #include #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/eloq_locker_noop.h" #include "mongo/db/concurrency/global_lock_acquisition_tracker.h" #include "mongo/db/concurrency/lock_manager_test_help.h" #include "mongo/db/concurrency/write_conflict_exception.h" @@ -123,6 +124,43 @@ class DConcurrencyTestFixture : public ServiceContextMongoDTest { }; +TEST_F(DConcurrencyTestFixture, FixtureInstallsDefaultLockerAcrossPoolReuse) { + // Use Client directly so the fixture must cover every operation-context checkout, + // not just calls to the fixture's makeOperationContext() helper. + auto client = getServiceContext()->makeClient("fixture_locker_reuse"); + for (int i = 0; i < 64; ++i) { + auto opCtx = client->makeOperationContext(); + ASSERT_TRUE(dynamic_cast(opCtx->lockState())); + ASSERT_TRUE(opCtx->recoveryUnit()); + Lock::GlobalLock lock(opCtx.get(), MODE_IX); + ASSERT_TRUE(opCtx->lockState()->isLocked()); + } +} + +TEST_F(DConcurrencyTestFixture, FixtureLockerDoesNotLeakIntoBareServiceContext) { + // Populate the thread-local pool with fixture contexts using real MongoDB lockers. + for (int i = 0; i < 64; ++i) { + auto opCtx = makeOperationContext(); + ASSERT_TRUE(dynamic_cast(opCtx->lockState())); + } + + auto bareService = ServiceContext::make(); + auto bareClient = bareService->makeClient("bare_locker_reuse"); + ASSERT_FALSE(bareService->getStorageEngine()); + for (int i = 0; i < 64; ++i) { + auto opCtx = bareClient->makeOperationContext(); + ASSERT_TRUE(dynamic_cast(opCtx->lockState())); + ASSERT_FALSE(opCtx->recoveryUnit()); + } + + // Returning to the fixture must reinstall its real locker after production reuse. + for (int i = 0; i < 64; ++i) { + auto opCtx = makeOperationContext(); + ASSERT_TRUE(dynamic_cast(opCtx->lockState())); + ASSERT_TRUE(opCtx->recoveryUnit()); + } +} + TEST_F(DConcurrencyTestFixture, WriteConflictRetryInstantiatesOK) { auto opCtx = makeOperationContext(); opCtx->swapLockState(stdx::make_unique()); diff --git a/src/mongo/db/coro_sync.cpp b/src/mongo/db/coro_sync.cpp index dc6d903b90..8c6039cbd8 100644 --- a/src/mongo/db/coro_sync.cpp +++ b/src/mongo/db/coro_sync.cpp @@ -42,31 +42,22 @@ void Mutex::lock() { void ConditionVariable::wait(std::unique_lock& lock) { invariant(lock.owns_lock()); - if (LocalThread::ID() != -1) { - Client* client = Client::getCurrent(); - if (client) { - const CoroutineFunctors& coro = Client::getCurrent()->coroutineFunctors(); - if (coro != CoroutineFunctors::Unavailable) { - lock.unlock(); - (*coro.longResumeFuncPtr)(); - (*coro.yieldFuncPtr)(); - lock.lock(); - } else { - MONGO_LOG(1) << "ThreadGroup " << LocalThread::ID() - << " call std::condition_variable::wait because the coroutine context " - "is unavailable."; - _cv.wait(reinterpret_cast&>(lock)); - } - } else { - MONGO_LOG(1) - << "ThreadGroup " << LocalThread::ID() - << " call std::condition_variable::wait because the client object is unavailable."; - _cv.wait(reinterpret_cast&>(lock)); - } - - } else { + if (!_yieldIfCoroutine(lock)) { _cv.wait(reinterpret_cast&>(lock)); } } + +bool ConditionVariable::_yieldIfCoroutine(std::unique_lock& lock) { + if (LocalThread::ID() == -1 || !Client::getCurrent()) + return false; + const auto& coro = Client::getCurrent()->coroutineFunctors(); + if (coro == CoroutineFunctors::Unavailable) + return false; + lock.unlock(); + (*coro.longResumeFuncPtr)(); + (*coro.yieldFuncPtr)(); + lock.lock(); + return true; +} } // namespace coro } // namespace mongo diff --git a/src/mongo/db/coro_sync.h b/src/mongo/db/coro_sync.h index 6c5cb78f9c..ce6d6f98e7 100644 --- a/src/mongo/db/coro_sync.h +++ b/src/mongo/db/coro_sync.h @@ -74,8 +74,14 @@ class ConditionVariable { template std::cv_status wait_until(std::unique_lock& lock, const std::chrono::time_point& timeout_time) { - wait(lock); - return Clock::now() < timeout_time ? std::cv_status::no_timeout : std::cv_status::timeout; + if (Clock::now() >= timeout_time) + return std::cv_status::timeout; + if (_yieldIfCoroutine(lock)) { + return Clock::now() < timeout_time ? std::cv_status::no_timeout + : std::cv_status::timeout; + } + return _cv.wait_until(reinterpret_cast&>(lock), + timeout_time); } template @@ -105,11 +111,11 @@ class ConditionVariable { } std::cv_status wait_until(std::unique_lock& lock, Date_t timeout_time) { - wait(lock); - return Date_t::now() < timeout_time ? std::cv_status::no_timeout : std::cv_status::timeout; + return wait_until(lock, timeout_time.toSystemTimePoint()); } private: + bool _yieldIfCoroutine(std::unique_lock& lock); std::condition_variable _cv; }; } // namespace coro diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 7f148932ab..61f5b24958 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -92,6 +92,7 @@ #include "mongo/db/logical_time_validator.h" #include "mongo/db/mongod_options.h" #include "mongo/db/op_observer_registry.h" +#include "mongo/db/op_observer_impl.h" #include "mongo/db/operation_context.h" #include "mongo/db/periodic_runner_job_abort_expired_transactions.h" #include "mongo/db/query/internal_plans.h" @@ -99,14 +100,19 @@ #include "mongo/db/repl/drop_pending_collection_reaper.h" #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/repl_settings.h" -#include "mongo/db/repl/replication_consistency_markers_impl.h" #include "mongo/db/repl/replication_coordinator.h" +#ifdef ELOQDOC_STANDALONE +#include "mongo/db/repl/replication_coordinator_standalone.h" +#else +#include "mongo/db/repl/replication_consistency_markers_impl.h" #include "mongo/db/repl/replication_coordinator_external_state_impl.h" #include "mongo/db/repl/replication_coordinator_impl.h" -#include "mongo/db/repl/replication_process.h" #include "mongo/db/repl/replication_recovery.h" -#include "mongo/db/repl/storage_interface_impl.h" #include "mongo/db/repl/topology_coordinator.h" +#endif +#include "mongo/db/repl/replication_process.h" +#include "mongo/db/repl/storage_interface_impl.h" +#ifndef ELOQDOC_STANDALONE #include "mongo/db/s/balancer/balancer.h" #include "mongo/db/s/config/sharding_catalog_manager.h" #include "mongo/db/s/config_server_op_observer.h" @@ -114,6 +120,7 @@ #include "mongo/db/s/shard_server_op_observer.h" #include "mongo/db/s/sharding_initialization_mongod.h" #include "mongo/db/s/sharding_state_recovery.h" +#endif #include "mongo/db/server_options.h" #include "mongo/db/server_parameters.h" #include "mongo/db/service_context.h" @@ -286,11 +293,13 @@ ExitCode _initAndListen(int listenPort) { opObserverRegistry->addObserver(stdx::make_unique()); +#ifndef ELOQDOC_STANDALONE if (serverGlobalParams.clusterRole == ClusterRole::ShardServer) { opObserverRegistry->addObserver(stdx::make_unique()); } else if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) { opObserverRegistry->addObserver(stdx::make_unique()); } +#endif setupFreeMonitoringOpObserver(opObserverRegistry.get()); @@ -525,12 +534,14 @@ ExitCode _initAndListen(int listenPort) { << startupWarningsLog; } +#ifndef ELOQDOC_STANDALONE // This function may take the global lock. auto shardingInitialized = ShardingInitializationMongoD::get(startupOpCtx.get()) ->initializeShardingAwarenessIfNeeded(startupOpCtx.get()); if (shardingInitialized) { waitForShardRegistryReload(startupOpCtx.get()).transitional_ignore(); } +#endif auto storageEngine = serviceContext->getStorageEngine(); invariant(storageEngine); @@ -547,6 +558,7 @@ ExitCode _initAndListen(int listenPort) { restartInProgressIndexesFromLastShutdown(startupOpCtx.get()); +#ifndef ELOQDOC_STANDALONE if (serverGlobalParams.clusterRole == ClusterRole::ShardServer) { // Note: For replica sets, ShardingStateRecovery happens on transition to primary. if (!repl::ReplicationCoordinator::get(startupOpCtx.get())->isReplEnabled()) { @@ -573,6 +585,7 @@ ExitCode _initAndListen(int listenPort) { LogicalTimeValidator::set(startupOpCtx->getServiceContext(), stdx::make_unique(keyManager)); } +#endif repl::ReplicationCoordinator::get(startupOpCtx.get())->startup(startupOpCtx.get()); const unsigned long long missingRepl = @@ -801,6 +814,7 @@ void startupConfigActions(const std::vector& args) { #endif } +#ifndef ELOQDOC_STANDALONE auto makeReplicationExecutor(ServiceContext* serviceContext) { ThreadPool::Options tpOptions; tpOptions.poolName = "replexec"; @@ -814,9 +828,23 @@ auto makeReplicationExecutor(ServiceContext* serviceContext) { stdx::make_unique(tpOptions), executor::makeNetworkInterface("Replication", nullptr, std::move(hookList))); } +#endif void setUpReplication(ServiceContext* serviceContext) { repl::StorageInterface::set(serviceContext, stdx::make_unique()); +#ifdef ELOQDOC_STANDALONE + // Preserve the server startup path, but omit MongoDB replica-set services. Eloq's + // transaction and durability services are initialized by the storage engine. + // Unsupported topology options are rejected by storeMongodOptions before startup. + // ReplicationProcess and DropPendingCollectionReaper are deliberately not installed. + // Callers must check replication mode before accessing either service; keep this + // contract when merging upstream startup, catalog, and shutdown changes. + invariant(!getGlobalReplSettings().usingReplSets() && + serverGlobalParams.clusterRole == ClusterRole::None); + LogicalClock::set(serviceContext, stdx::make_unique(serviceContext)); + repl::ReplicationCoordinator::set( + serviceContext, stdx::make_unique(serviceContext)); +#else auto storageInterface = repl::StorageInterface::get(serviceContext); auto consistencyMarkers = @@ -851,6 +879,7 @@ void setUpReplication(ServiceContext* serviceContext) { storageInterface, static_cast(curTimeMillis64())); repl::ReplicationCoordinator::set(serviceContext, std::move(replCoord)); +#endif repl::setOplogCollectionName(serviceContext); } @@ -895,7 +924,9 @@ void shutdownTask() { // is building an index. repl::ReplicationCoordinator::get(serviceContext)->shutdown(opCtx); +#ifndef ELOQDOC_STANDALONE ShardingInitializationMongoD::get(serviceContext)->shutDown(opCtx); +#endif // Destroy all stashed transaction resources, in order to release locks. SessionKiller::Matcher matcherAllSessions( @@ -912,9 +943,11 @@ void shutdownTask() { ReplicaSetMonitor::shutdown(); +#ifndef ELOQDOC_STANDALONE if (auto sr = Grid::get(serviceContext)->shardRegistry()) { sr->shutdown(); } +#endif // Validator shutdown must be called after setKillAllOperations is called. Otherwise, this can // deadlock. diff --git a/src/mongo/db/ftdc/ftdc_test.cpp b/src/mongo/db/ftdc/ftdc_test.cpp index 631ca23397..4cdeb80ed8 100644 --- a/src/mongo/db/ftdc/ftdc_test.cpp +++ b/src/mongo/db/ftdc/ftdc_test.cpp @@ -36,9 +36,11 @@ #include "mongo/base/init.h" #include "mongo/bson/bson_validate.h" #include "mongo/db/client.h" +#include "mongo/db/concurrency/locker_noop.h" #include "mongo/db/ftdc/file_reader.h" #include "mongo/db/jsobj.h" #include "mongo/db/service_context.h" +#include "mongo/db/storage/recovery_unit_noop.h" #include "mongo/stdx/memory.h" #include "mongo/unittest/unittest.h" #include "mongo/util/clock_source.h" @@ -49,6 +51,24 @@ namespace mongo { namespace { +// FTDC's background collector creates operation contexts directly through its Client. +// Install mock storage state for this engine-less fixture, including after pool reuse. +class FTDCStorageObserver final : public ServiceContext::ClientObserver { +public: + void onCreateClient(Client*) override {} + void onDestroyClient(Client*) override {} + void onCreateOperationContext(OperationContext* opCtx) override { + if (!opCtx->lockState()) { + opCtx->setLockState(stdx::make_unique()); + } + if (!opCtx->recoveryUnit()) { + opCtx->setRecoveryUnit(new RecoveryUnitNoop(), + WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); + } + } + void onDestroyOperationContext(OperationContext*) override {} +}; + BSONObj filteredFTDCCopy(const BSONObj& obj) { BSONObjBuilder builder; for (const auto& f : obj) { @@ -138,11 +158,20 @@ void createDirectoryClean(const boost::filesystem::path& dir) { boost::filesystem::create_directory(dir); } -FTDCTest::FTDCTest() { +FTDCTest::FTDCTest() : ServiceContextTest(stdx::make_unique()) { auto service = getServiceContext(); service->setFastClockSource(stdx::make_unique()); service->setPreciseClockSource(stdx::make_unique()); service->setTickSource(stdx::make_unique()); } +TEST_F(FTDCTest, InstallsNoopStorageStateAcrossPoolReuse) { + ASSERT_FALSE(getServiceContext()->getStorageEngine()); + for (int i = 0; i < 64; ++i) { + auto opCtx = makeOperationContext(); + ASSERT_TRUE(dynamic_cast(opCtx->lockState())); + ASSERT_TRUE(dynamic_cast(opCtx->recoveryUnit())); + } +} + } // namespace mongo diff --git a/src/mongo/db/logical_session_cache_factory_standalone.cpp b/src/mongo/db/logical_session_cache_factory_standalone.cpp new file mode 100644 index 0000000000..7af0e88b81 --- /dev/null +++ b/src/mongo/db/logical_session_cache_factory_standalone.cpp @@ -0,0 +1,25 @@ +// CMake-only implementation of the normal mongod logical-session-cache factory. +// The SCons build continues to use logical_session_cache_factory_mongod.cpp. + +#include "mongo/platform/basic.h" + +#include "mongo/db/logical_session_cache_factory_mongod.h" + +#include "mongo/db/logical_session_cache_impl.h" +#include "mongo/db/service_liaison_mongod.h" +#include "mongo/db/sessions_collection_standalone.h" +#include "mongo/stdx/memory.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + +std::unique_ptr makeLogicalSessionCacheD(LogicalSessionCacheServer state) { + invariant(state == LogicalSessionCacheServer::kStandalone); + return stdx::make_unique( + stdx::make_unique(), + std::make_shared(), + nullptr, // MongoDB's transaction reaper is only used by replica sets and shards. + LogicalSessionCacheImpl::Options{}); +} + +} // namespace mongo diff --git a/src/mongo/db/logical_session_id_test.cpp b/src/mongo/db/logical_session_id_test.cpp index e32e1e8808..08dcc17045 100644 --- a/src/mongo/db/logical_session_id_test.cpp +++ b/src/mongo/db/logical_session_id_test.cpp @@ -292,21 +292,22 @@ TEST_F(LogicalSessionIdTest, InitializeOperationSessionInfo_SessionIdAndTransact ASSERT_EQ(100, *_opCtx->getTxnNumber()); } -TEST_F(LogicalSessionIdTest, InitializeOperationSessionInfo_IsReplSetMemberOrMongosFalse) { +TEST_F(LogicalSessionIdTest, InitializeOperationSessionInfo_StandaloneSupportsTransactionNumber) { addSimpleUser(UserName("simple", "test")); LogicalSessionFromClient lsid; lsid.setId(UUID::gen()); - ASSERT_THROWS_CODE( - initializeOperationSessionInfo( - _opCtx.get(), - BSON("TestCmd" << 1 << "lsid" << lsid.toBSON() << "txnNumber" << 100LL << "OtherField" - << "TestField"), - true, - false, - true), - AssertionException, - ErrorCodes::IllegalOperation); + // Eloq provides transactions through its storage engine without MongoDB replication/sharding. + initializeOperationSessionInfo( + _opCtx.get(), + BSON("TestCmd" << 1 << "lsid" << lsid.toBSON() << "txnNumber" << 100LL), + true, + false, + true); + ASSERT(_opCtx->getLogicalSessionId()); + ASSERT_EQ(lsid.getId(), _opCtx->getLogicalSessionId()->getId()); + ASSERT(_opCtx->getTxnNumber()); + ASSERT_EQ(100, *_opCtx->getTxnNumber()); } TEST_F(LogicalSessionIdTest, InitializeOperationSessionInfo_SupportsDocLockingFalse) { diff --git a/src/mongo/db/matcher/expression_optimize_test.cpp b/src/mongo/db/matcher/expression_optimize_test.cpp index 036a1146b2..4327fb6706 100644 --- a/src/mongo/db/matcher/expression_optimize_test.cpp +++ b/src/mongo/db/matcher/expression_optimize_test.cpp @@ -77,7 +77,7 @@ Status isValid(const std::string& queryStr, const QueryRequest& qrRaw) { TEST(ExpressionOptimizeTest, IsValidText) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); ASSERT_OK(qr->validate()); // Valid: regular TEXT. @@ -133,7 +133,7 @@ TEST(ExpressionOptimizeTest, IsValidText) { TEST(ExpressionOptimizeTest, IsValidTextTailable) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setTailableMode(TailableModeEnum::kTailable); ASSERT_OK(qr->validate()); @@ -143,7 +143,7 @@ TEST(ExpressionOptimizeTest, IsValidTextTailable) { TEST(ExpressionOptimizeTest, IsValidGeo) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); ASSERT_OK(qr->validate()); // Valid: regular GEO_NEAR. @@ -209,7 +209,7 @@ TEST(ExpressionOptimizeTest, IsValidGeo) { TEST(ExpressionOptimizeTest, IsValidTextAndGeo) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); ASSERT_OK(qr->validate()); // Invalid: TEXT and GEO_NEAR. @@ -230,7 +230,7 @@ TEST(ExpressionOptimizeTest, IsValidTextAndGeo) { TEST(ExpressionOptimizeTest, IsValidTextAndNaturalAscending) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setSort(fromjson("{$natural: 1}")); ASSERT_OK(qr->validate()); @@ -240,7 +240,7 @@ TEST(ExpressionOptimizeTest, IsValidTextAndNaturalAscending) { TEST(ExpressionOptimizeTest, IsValidTextAndNaturalDescending) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setSort(fromjson("{$natural: -1}")); ASSERT_OK(qr->validate()); @@ -250,7 +250,7 @@ TEST(ExpressionOptimizeTest, IsValidTextAndNaturalDescending) { TEST(ExpressionOptimizeTest, IsValidTextAndHint) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setHint(fromjson("{a: 1}")); ASSERT_OK(qr->validate()); @@ -261,7 +261,7 @@ TEST(ExpressionOptimizeTest, IsValidTextAndHint) { // SERVER-14366 TEST(ExpressionOptimizeTest, IsValidGeoNearNaturalSort) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setSort(fromjson("{$natural: 1}")); ASSERT_OK(qr->validate()); @@ -272,7 +272,7 @@ TEST(ExpressionOptimizeTest, IsValidGeoNearNaturalSort) { // SERVER-14366 TEST(ExpressionOptimizeTest, IsValidGeoNearNaturalHint) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setHint(fromjson("{$natural: 1}")); ASSERT_OK(qr->validate()); diff --git a/src/mongo/db/modules/eloq/CMakeLists.txt b/src/mongo/db/modules/eloq/CMakeLists.txt index 65e9b5e076..4c2e92bfbf 100644 --- a/src/mongo/db/modules/eloq/CMakeLists.txt +++ b/src/mongo/db/modules/eloq/CMakeLists.txt @@ -47,3 +47,10 @@ endif() # --- Include build system modules --- add_subdirectory(data_substrate) + +# The pinned yaml-cpp 0.7.0 uses fixed-width integers in emitterutils.cpp without +# including . GCC 15 no longer supplies it through unrelated headers. +# Keep the compatibility include local to this downloaded dependency; this module +# is shared by the top-level CMake and historical SCons dependency builds. +target_compile_options(yaml-cpp PRIVATE + "$<$:SHELL:-include cstdint>") diff --git a/src/mongo/db/mongod_options.cpp b/src/mongo/db/mongod_options.cpp index 9eb6fe90ab..575a7a4e56 100644 --- a/src/mongo/db/mongod_options.cpp +++ b/src/mongo/db/mongod_options.cpp @@ -875,6 +875,18 @@ Status storeMongodOptions(const moe::Environment& params) { storageGlobalParams.engine = params["storage.engine"].as(); storageGlobalParams.engineSetByUser = true; } +#ifdef ELOQDOC_STANDALONE + if (params.count("replication.replSetName") || params.count("replication.replSet") || + params.count("sharding.clusterRole")) { + return Status(ErrorCodes::InvalidOptions, + "This EloqDoc build supports standalone mode only; --replSet, --shardsvr " + "and --configsvr are not supported"); + } + // The CMake server retains normal option parsing, but only the Eloq engine is linked. + if (!storageGlobalParams.engineSetByUser) { + storageGlobalParams.engine = "eloq"; + } +#endif if (params.count("storage.dbPath")) { storageGlobalParams.dbpath = params["storage.dbPath"].as(); diff --git a/src/mongo/db/nesting_depth_test.cpp b/src/mongo/db/nesting_depth_test.cpp index b6d325d299..1d1f2076ab 100644 --- a/src/mongo/db/nesting_depth_test.cpp +++ b/src/mongo/db/nesting_depth_test.cpp @@ -35,19 +35,55 @@ #include "mongo/bson/json.h" #include "mongo/client/connection_string.h" #include "mongo/executor/network_interface_integration_fixture.h" +#include "mongo/rpc/get_status_from_command_result.h" +#include "mongo/unittest/bson_test_util.h" #include "mongo/util/concurrency/thread_pool.h" namespace mongo { namespace executor { namespace { +constexpr auto kCollectionName = "depthTest"; + class NestingDepthFixture : public NetworkInterfaceIntegrationFixture { public: void setUp() final { startNet(); + RemoteCommandRequest request{fixture().getServers()[0], "admin", BSON("serverStatus" << 1), + BSONObj(), nullptr}; + auto response = runCommandSync(request); + ASSERT_OK(response.status); + ASSERT_OK(getStatusFromCommandResult(response.data)); + _usesEloq = response.data["storageEngine"].Obj()["name"].String() == "eloq"; + assertCommandOK(kCollectionName, BSON("dropDatabase" << 1)); } -}; -constexpr auto kCollectionName = "depthTest"; + void assertUpdateDepthError(const BSONObj& command) { + auto before = readDocument(); + if (_usesEloq) { + // Eloq's command-level transaction aborts the whole update command on error. + // Do not swallow that error into writeErrors and commit a partial transaction. + assertCommandFailsOnServer(kCollectionName, command, ErrorCodes::Overflow); + } else { + assertWriteError(kCollectionName, command, ErrorCodes::Overflow); + } + ASSERT_BSONOBJ_EQ(before, readDocument()); + } + +private: + BSONObj readDocument() { + RemoteCommandRequest request{fixture().getServers()[0], kCollectionName, + BSON("find" << kCollectionName << "singleBatch" << true), + BSONObj(), nullptr}; + auto response = runCommandSync(request); + ASSERT_OK(response.status); + ASSERT_OK(getStatusFromCommandResult(response.data)); + auto batch = response.data["cursor"].Obj()["firstBatch"].Array(); + ASSERT_EQ(1U, batch.size()); + return batch.front().Obj().getOwned(); + } + + bool _usesEloq = false; +}; /** * Appends an object to 'builder' that is nested 'depth' levels deep. @@ -274,7 +310,7 @@ TEST_F(NestingDepthFixture, CannotUpdateDocumentToExceedDepthLimit) { BSONObjBuilder updateCmd; appendUpdateCommandWithNestedDocuments( &updateCmd, largeButValidDepth, BSONDepth::getMaxDepthForUserStorage() + 1); - assertWriteError(kCollectionName, updateCmd.obj(), ErrorCodes::Overflow); + assertUpdateDepthError(updateCmd.obj()); } TEST_F(NestingDepthFixture, CanReplaceDocumentIfItStaysWithinDepthLimit) { @@ -308,7 +344,7 @@ TEST_F(NestingDepthFixture, CannotReplaceDocumentToExceedDepthLimit) { BSONObjBuilder updateCmd; appendUpdateReplaceCommandWithNestedDocuments( &updateCmd, largeButValidDepth, BSONDepth::getMaxDepthForUserStorage() + 1); - assertWriteError(kCollectionName, updateCmd.obj(), ErrorCodes::Overflow); + assertUpdateDepthError(updateCmd.obj()); } /** @@ -435,7 +471,7 @@ TEST_F(NestingDepthFixture, CannotUpdateArrayToExceedDepthLimit) { BSONObjBuilder updateCmd; appendUpdateCommandWithNestedArrays( &updateCmd, largeButValidDepth, BSONDepth::getMaxDepthForUserStorage() + 1); - assertWriteError(kCollectionName, updateCmd.obj(), ErrorCodes::Overflow); + assertUpdateDepthError(updateCmd.obj()); } TEST_F(NestingDepthFixture, CanReplaceArrayIfItStaysWithinDepthLimit) { @@ -469,7 +505,7 @@ TEST_F(NestingDepthFixture, CannotReplaceArrayToExceedDepthLimit) { BSONObjBuilder updateCmd; appendUpdateReplaceCommandWithNestedArrays( &updateCmd, largeButValidDepth, BSONDepth::getMaxDepthForUserStorage() + 1); - assertWriteError(kCollectionName, updateCmd.obj(), ErrorCodes::Overflow); + assertUpdateDepthError(updateCmd.obj()); } } // namespace } // namespace executor diff --git a/src/mongo/db/op_observer_noop.h b/src/mongo/db/op_observer_noop.h index 9e1a496660..f12830917a 100644 --- a/src/mongo/db/op_observer_noop.h +++ b/src/mongo/db/op_observer_noop.h @@ -61,7 +61,7 @@ class OpObserverNoop : public OpObserver { const BSONObj& msgObj, const boost::optional o2MsgObj) override {} void onCreateCollection(OperationContext* opCtx, - Collection* coll, + Collection::Uptr coll, const NamespaceString& collectionName, const CollectionOptions& options, const BSONObj& idIndex, diff --git a/src/mongo/db/operation_context.cpp b/src/mongo/db/operation_context.cpp index f7fc4ce5f6..0941629cd1 100644 --- a/src/mongo/db/operation_context.cpp +++ b/src/mongo/db/operation_context.cpp @@ -386,9 +386,7 @@ StatusWith OperationContext::waitForConditionOrInterruptNoAsser cv.wait(m); return stdx::cv_status::no_timeout; } - // return getServiceContext()->getPreciseClockSource()->waitForConditionUntil(cv, m, - // deadline); - return cv.wait_until(m, deadline); + return getServiceContext()->getPreciseClockSource()->waitForConditionUntil(cv, m, deadline); }(); // Continue waiting on cv until no other thread is attempting to kill this one. diff --git a/src/mongo/db/operation_context_test.cpp b/src/mongo/db/operation_context_test.cpp index b779102b70..f733a10b0d 100644 --- a/src/mongo/db/operation_context_test.cpp +++ b/src/mongo/db/operation_context_test.cpp @@ -29,12 +29,14 @@ #include "mongo/platform/basic.h" #include +#include #include "mongo/db/client.h" #include "mongo/db/json.h" #include "mongo/db/logical_session_id.h" #include "mongo/db/operation_context.h" #include "mongo/db/operation_context_group.h" +#include "mongo/db/operation_time_tracker.h" #include "mongo/db/service_context.h" #include "mongo/stdx/future.h" #include "mongo/stdx/memory.h" @@ -49,6 +51,27 @@ namespace mongo { namespace { +TEST(CoroutineConditionVariable, NativeThreadWaitForExpiresWithoutNotification) { + coro::Mutex mutex; + coro::ConditionVariable cv; + stdx::unique_lock lock(mutex); + const auto started = std::chrono::steady_clock::now(); + const auto delay = std::chrono::milliseconds(5); + ASSERT_FALSE(cv.wait_for(lock, delay, [] { return false; })); + ASSERT_TRUE(std::chrono::steady_clock::now() >= started + delay); + ASSERT_TRUE(lock.owns_lock()); +} + +TEST(CoroutineConditionVariable, NativeThreadDateDeadlineExpiresWithoutNotification) { + coro::Mutex mutex; + coro::ConditionVariable cv; + stdx::unique_lock lock(mutex); + auto deadline = Date_t::now() + Milliseconds(5); + while (cv.wait_until(lock, deadline) != stdx::cv_status::timeout) {} + ASSERT_TRUE(Date_t::now() >= deadline); + ASSERT_TRUE(lock.owns_lock()); +} + using unittest::assertGet; std::ostream& operator<<(std::ostream& os, stdx::cv_status cvStatus) { @@ -84,6 +107,79 @@ TEST(OperationContextTest, NoSessionIdNoTransactionNumber) { ASSERT(!opCtx->getTxnNumber()); } +TEST(OperationContextTest, NoStorageEngineLeavesRecoveryUnitUnset) { + auto serviceCtx = ServiceContext::make(); + auto client = serviceCtx->makeClient("OperationContextTest"); + ASSERT_FALSE(serviceCtx->getStorageEngine()); + // Exercise creation and repeated pool return/reuse without a test storage observer. + for (int i = 0; i < 64; ++i) { + auto opCtx = client->makeOperationContext(); + ASSERT_FALSE(opCtx->recoveryUnit()); + } +} + +TEST(OperationContextTest, PoolReleaseReusesUnsharedOperationTimeTracker) { + auto serviceCtx = ServiceContext::make(); + auto client = serviceCtx->makeClient("OperationContextTest"); + auto opCtx = client->makeOperationContext(); + auto tracker = OperationTimeTracker::get(opCtx.get()); + auto* original = tracker.get(); + // A weak reference observes lifetime without preventing exclusive-ownership reuse. + std::weak_ptr lifetime = tracker; + tracker->updateOperationTime(LogicalTime(Timestamp(15))); + tracker.reset(); + + for (int i = 0; i < 10; ++i) { + ASSERT_EQ(1, lifetime.use_count()); + opCtx->resetAllDecorations(); + // This also rules out destroy/reallocate-at-the-same-address implementations. + ASSERT_FALSE(lifetime.expired()); + auto reused = OperationTimeTracker::get(opCtx.get()); + ASSERT_EQ(original, reused.get()); + ASSERT_EQ(LogicalTime::kUninitialized, reused->getMaxOperationTime()); + reused->updateOperationTime(LogicalTime(Timestamp(5))); + ASSERT_EQ(LogicalTime(Timestamp(5)), reused->getMaxOperationTime()); + } + + // Exercise the actual ServiceContext deleter -> pool -> decoration-reset path too. + opCtx.reset(); + ASSERT_FALSE(lifetime.expired()); + ASSERT_EQ(LogicalTime::kUninitialized, lifetime.lock()->getMaxOperationTime()); +} + +TEST(OperationContextTest, PoolReleasePreservesSharedOperationTimeTracker) { + auto serviceCtx = ServiceContext::make(); + auto client = serviceCtx->makeClient("OperationContextTest"); + auto opCtx = client->makeOperationContext(); + auto previous = OperationTimeTracker::get(opCtx.get()); + const LogicalTime previousTime(Timestamp(15)); + previous->updateOperationTime(previousTime); + + opCtx->resetAllDecorations(); + auto current = OperationTimeTracker::get(opCtx.get()); + ASSERT_NE(previous, current); + ASSERT_EQ(previousTime, previous->getMaxOperationTime()); + ASSERT_EQ(LogicalTime::kUninitialized, current->getMaxOperationTime()); + previous->updateOperationTime(LogicalTime(Timestamp(30))); + ASSERT_EQ(LogicalTime::kUninitialized, current->getMaxOperationTime()); + current->updateOperationTime(LogicalTime(Timestamp(5))); + ASSERT_EQ(LogicalTime(Timestamp(30)), previous->getMaxOperationTime()); + + // Once the replacement is exclusively owned, subsequent recycling must reuse it. + std::weak_ptr lifetime = current; + current.reset(); + opCtx->resetAllDecorations(); + ASSERT_FALSE(lifetime.expired()); + ASSERT_EQ(LogicalTime::kUninitialized, lifetime.lock()->getMaxOperationTime()); + + // A retained tracker also survives the real pool-return path with its state intact. + current = OperationTimeTracker::get(opCtx.get()); + current->updateOperationTime(LogicalTime(Timestamp(7))); + opCtx.reset(); + ASSERT_EQ(LogicalTime(Timestamp(7)), current->getMaxOperationTime()); + ASSERT_EQ(LogicalTime(Timestamp(30)), previous->getMaxOperationTime()); +} + TEST(OperationContextTest, SessionIdNoTransactionNumber) { auto serviceCtx = ServiceContext::make(); auto client = serviceCtx->makeClient("OperationContextTest"); diff --git a/src/mongo/db/operation_time_tracker.cpp b/src/mongo/db/operation_time_tracker.cpp index 7bf6f97f24..2bec454783 100644 --- a/src/mongo/db/operation_time_tracker.cpp +++ b/src/mongo/db/operation_time_tracker.cpp @@ -36,7 +36,14 @@ namespace { struct OperationTimeTrackerHolder { OperationTimeTrackerHolder() : opTimeTracker(std::make_shared()) {} void reset() { - opTimeTracker.reset(); + // OperationContext recycling has exclusive access to this holder. Reuse the tracker + // unless another shared owner (e.g. a remote callback) still needs the previous state. + invariant(opTimeTracker); + if (opTimeTracker.use_count() == 1) { + opTimeTracker->reset(); + } else { + opTimeTracker = std::make_shared(); + } } static const OperationContext::Decoration get; std::shared_ptr opTimeTracker; @@ -47,7 +54,7 @@ const OperationContext::Decoration OperationTimeTrac } // namespace std::shared_ptr OperationTimeTracker::get(OperationContext* opCtx) { - auto timeTrackerHolder = OperationTimeTrackerHolder::get(opCtx); + auto& timeTrackerHolder = OperationTimeTrackerHolder::get(opCtx); invariant(timeTrackerHolder.opTimeTracker); return timeTrackerHolder.opTimeTracker; } @@ -64,4 +71,9 @@ void OperationTimeTracker::updateOperationTime(LogicalTime newTime) { } } +void OperationTimeTracker::reset() { + stdx::lock_guard lock(_mutex); + _maxOperationTime = LogicalTime::kUninitialized; +} + } // namespace mongo diff --git a/src/mongo/db/operation_time_tracker.h b/src/mongo/db/operation_time_tracker.h index d1479776ad..b8c62a11f0 100644 --- a/src/mongo/db/operation_time_tracker.h +++ b/src/mongo/db/operation_time_tracker.h @@ -54,6 +54,9 @@ class OperationTimeTracker { */ void updateOperationTime(LogicalTime newTime); + // Clear state for reuse only when no other operation/callback owns this tracker. + void reset(); + private: // protects _maxOperationTime mutable stdx::mutex _mutex; diff --git a/src/mongo/db/operation_time_tracker_test.cpp b/src/mongo/db/operation_time_tracker_test.cpp index f582c891c6..39d823a1ef 100644 --- a/src/mongo/db/operation_time_tracker_test.cpp +++ b/src/mongo/db/operation_time_tracker_test.cpp @@ -30,11 +30,31 @@ #include "mongo/db/logical_time.h" #include "mongo/db/operation_time_tracker.h" +#include "mongo/db/service_context_test_fixture.h" #include "mongo/unittest/unittest.h" namespace mongo { namespace { +using OperationTimeTrackerTest = ServiceContextTest; + +TEST_F(OperationTimeTrackerTest, ResetDecorationsCreatesIndependentTracker) { + auto opCtx = makeOperationContext(); + auto previous = OperationTimeTracker::get(opCtx.get()); + const LogicalTime previousTime(Timestamp(15)); + previous->updateOperationTime(previousTime); + ASSERT_EQ(previous, OperationTimeTracker::get(opCtx.get())); + + // OperationContext's pool deleter resets decorations without destroying the context. + opCtx->resetAllDecorations(); + auto current = OperationTimeTracker::get(opCtx.get()); + ASSERT_NE(previous, current); + ASSERT_EQ(LogicalTime::kUninitialized, current->getMaxOperationTime()); + current->updateOperationTime(LogicalTime(Timestamp(5))); + ASSERT_EQ(previousTime, previous->getMaxOperationTime()); + ASSERT_EQ(current, OperationTimeTracker::get(opCtx.get())); +} + TEST(OperationTimeTracker, UnintializedMaxOperationTime) { OperationTimeTracker opTimeTracker; ASSERT_TRUE(opTimeTracker.getMaxOperationTime() == LogicalTime::kUninitialized); diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index fbdc846e83..6e08719ccf 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -307,6 +307,7 @@ pipelineeEnv.Library( 'document_source_unwind.cpp', 'mongo_process_common.cpp', 'pipeline.cpp', + 'pipeline_sharded.cpp', 'sequential_document_cache.cpp', 'tee_buffer.cpp', ], diff --git a/src/mongo/db/pipeline/aggregation_context_fixture.h b/src/mongo/db/pipeline/aggregation_context_fixture.h index 4f53f2fc55..003544f25f 100644 --- a/src/mongo/db/pipeline/aggregation_context_fixture.h +++ b/src/mongo/db/pipeline/aggregation_context_fixture.h @@ -51,7 +51,7 @@ class AggregationContextFixture : public ServiceContextTest { // Must instantiate ExpressionContext _after_ setting the TZ database on the service // context. // _expCtx = new ExpressionContext(_opCtx.get(), nullptr); - _expCtx = ObjectPool::newObjectRawPointer(_opCtx, nullptr); + _expCtx = ObjectPool::newObjectRawPointer(_opCtx.get(), nullptr); _expCtx->ns = std::move(nss); } diff --git a/src/mongo/db/pipeline/pipeline.cpp b/src/mongo/db/pipeline/pipeline.cpp index 12598b1c57..b8bf88cf70 100644 --- a/src/mongo/db/pipeline/pipeline.cpp +++ b/src/mongo/db/pipeline/pipeline.cpp @@ -38,12 +38,10 @@ #include "mongo/db/jsobj.h" #include "mongo/db/operation_context.h" #include "mongo/db/pipeline/accumulator.h" -#include "mongo/db/pipeline/cluster_aggregation_planner.h" #include "mongo/db/pipeline/document.h" #include "mongo/db/pipeline/document_source.h" #include "mongo/db/pipeline/document_source_geo_near.h" #include "mongo/db/pipeline/document_source_match.h" -#include "mongo/db/pipeline/document_source_merge_cursors.h" #include "mongo/db/pipeline/document_source_out.h" #include "mongo/db/pipeline/document_source_project.h" #include "mongo/db/pipeline/document_source_sort.h" @@ -314,25 +312,6 @@ void Pipeline::dispose(OperationContext* opCtx) { } } -std::unique_ptr Pipeline::splitForSharded() { - invariant(!isSplitForShards()); - invariant(!isSplitForMerge()); - - // Create and initialize the shard spec we'll return. We start with an empty pipeline on the - // shards and all work being done in the merger. Optimizations can move operations between - // the pipelines to be more efficient. - std::unique_ptr shardPipeline(new Pipeline(pCtx), - PipelineDeleter(pCtx->opCtx)); - - cluster_aggregation_planner::performSplitPipelineOptimizations(shardPipeline.get(), this); - shardPipeline->_splitState = SplitState::kSplitForShards; - _splitState = SplitState::kSplitForMerge; - - stitch(); - - return shardPipeline; -} - BSONObj Pipeline::getInitialQuery() const { if (_sources.empty()) return BSONObj(); diff --git a/src/mongo/db/pipeline/pipeline_sharded.cpp b/src/mongo/db/pipeline/pipeline_sharded.cpp new file mode 100644 index 0000000000..c673b8d121 --- /dev/null +++ b/src/mongo/db/pipeline/pipeline_sharded.cpp @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2011 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/pipeline/pipeline.h" + +#include "mongo/db/pipeline/cluster_aggregation_planner.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + +// Only builds with MongoDB sharding link this translation unit. The common Pipeline +// implementation can be linked without referencing the distributed aggregation planner. +std::unique_ptr Pipeline::splitForSharded() { + invariant(!isSplitForShards()); + invariant(!isSplitForMerge()); + + // Create and initialize the shard spec we'll return. We start with an empty pipeline on the + // shards and all work being done in the merger. Optimizations can move operations between + // the pipelines to be more efficient. + std::unique_ptr shardPipeline(new Pipeline(pCtx), + PipelineDeleter(pCtx->opCtx)); + + cluster_aggregation_planner::performSplitPipelineOptimizations(shardPipeline.get(), this); + shardPipeline->_splitState = SplitState::kSplitForShards; + _splitState = SplitState::kSplitForMerge; + + stitch(); + + return shardPipeline; +} + +} // namespace mongo diff --git a/src/mongo/db/query/canonical_query_test.cpp b/src/mongo/db/query/canonical_query_test.cpp index 8f7a35ad8a..3c41fb95a1 100644 --- a/src/mongo/db/query/canonical_query_test.cpp +++ b/src/mongo/db/query/canonical_query_test.cpp @@ -175,13 +175,13 @@ TEST(CanonicalQueryTest, SortTreeNumChildrenComparison) { /** * Utility function to create a CanonicalQuery */ -unique_ptr canonicalize(const char* queryStr, +CanonicalQuery::UPtr canonicalize(const char* queryStr, MatchExpressionParser::AllowedFeatureSet allowedFeatures = MatchExpressionParser::kDefaultSpecialFeatures) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); auto statusWithCQ = CanonicalQuery::canonicalize( @@ -191,13 +191,13 @@ unique_ptr canonicalize(const char* queryStr, return std::move(statusWithCQ.getValue()); } -std::unique_ptr canonicalize(const char* queryStr, +CanonicalQuery::UPtr canonicalize(const char* queryStr, const char* sortStr, const char* projStr) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projStr)); @@ -210,7 +210,7 @@ std::unique_ptr canonicalize(const char* queryStr, * Test function for CanonicalQuery::normalize. */ void testNormalizeQuery(const char* queryStr, const char* expectedExprStr) { - unique_ptr cq(canonicalize(queryStr)); + CanonicalQuery::UPtr cq(canonicalize(queryStr)); MatchExpression* me = cq->root(); BSONObj expectedExprObj = fromjson(expectedExprStr); unique_ptr expectedExpr(parseMatchExpression(expectedExprObj)); @@ -289,7 +289,7 @@ TEST(CanonicalQueryTest, CanonicalQueryFromQRWithNoCollation) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); auto cq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), std::move(qr))); ASSERT_TRUE(cq->getCollator() == nullptr); } @@ -298,7 +298,7 @@ TEST(CanonicalQueryTest, CanonicalQueryFromQRWithCollation) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setCollation(BSON("locale" << "reverse")); auto cq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), std::move(qr))); @@ -310,7 +310,7 @@ TEST(CanonicalQueryTest, CanonicalQueryFromBaseQueryWithNoCollation) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{$or:[{a:1,b:1},{a:1,c:1}]}")); auto baseCq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), std::move(qr))); MatchExpression* firstClauseExpr = baseCq->root()->getChild(0); @@ -323,7 +323,7 @@ TEST(CanonicalQueryTest, CanonicalQueryFromBaseQueryWithCollation) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{$or:[{a:1,b:1},{a:1,c:1}]}")); qr->setCollation(BSON("locale" << "reverse")); @@ -339,7 +339,7 @@ TEST(CanonicalQueryTest, SettingCollatorUpdatesCollatorAndMatchExpression) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 'foo', b: {$in: ['bar', 'baz']}}")); auto cq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), std::move(qr))); ASSERT_EQUALS(2U, cq->root()->numChildren()); @@ -368,7 +368,7 @@ TEST(CanonicalQueryTest, SettingCollatorUpdatesCollatorAndMatchExpression) { } TEST(CanonicalQueryTest, NorWithOneChildNormalizedToNot) { - unique_ptr cq(canonicalize("{$nor: [{a: 1}]}")); + CanonicalQuery::UPtr cq(canonicalize("{$nor: [{a: 1}]}")); auto root = cq->root(); ASSERT_EQ(MatchExpression::NOT, root->matchType()); ASSERT_EQ(1U, root->numChildren()); @@ -376,13 +376,13 @@ TEST(CanonicalQueryTest, NorWithOneChildNormalizedToNot) { } TEST(CanonicalQueryTest, NorWithTwoChildrenNotNormalized) { - unique_ptr cq(canonicalize("{$nor: [{a: 1}, {b: 1}]}")); + CanonicalQuery::UPtr cq(canonicalize("{$nor: [{a: 1}, {b: 1}]}")); auto root = cq->root(); ASSERT_EQ(MatchExpression::NOR, root->matchType()); } TEST(CanonicalQueryTest, NorWithOneChildNormalizedAfterNormalizingChild) { - unique_ptr cq(canonicalize("{$nor: [{$or: [{a: 1}]}]}")); + CanonicalQuery::UPtr cq(canonicalize("{$nor: [{$or: [{a: 1}]}]}")); auto root = cq->root(); ASSERT_EQ(MatchExpression::NOT, root->matchType()); ASSERT_EQ(1U, root->numChildren()); diff --git a/src/mongo/db/query/collation/SConscript b/src/mongo/db/query/collation/SConscript index 09f8043a4c..82f48f9d92 100644 --- a/src/mongo/db/query/collation/SConscript +++ b/src/mongo/db/query/collation/SConscript @@ -100,19 +100,7 @@ env.CppUnitTest( ], ) -env.CppUnitTest( - target="collator_factory_icu_test", - source=[ - "collator_factory_icu_locales_test.cpp", - "collator_factory_icu_test.cpp", - ], - LIBDEPS=[ - "collator_icu", - ], -) - -# The collator_icu library and the collator_interface_icu_test unit tests need an environment which -# has access to the third-party ICU headers. +# The collator library and both ICU test targets must use matching ICU headers and defines. icuEnv = env.Clone() if not use_system_version_of_library("icu"): @@ -148,6 +136,17 @@ icuEnv.Library( ], ) +icuEnv.CppUnitTest( + target="collator_factory_icu_test", + source=[ + "collator_factory_icu_locales_test.cpp", + "collator_factory_icu_test.cpp", + ], + LIBDEPS=[ + "collator_icu", + ], +) + icuEnv.CppUnitTest( target="collator_interface_icu_test", source=[ diff --git a/src/mongo/db/query/collation/collator_factory_icu_test.cpp b/src/mongo/db/query/collation/collator_factory_icu_test.cpp index e2a393a787..742c2e9957 100644 --- a/src/mongo/db/query/collation/collator_factory_icu_test.cpp +++ b/src/mongo/db/query/collation/collator_factory_icu_test.cpp @@ -30,6 +30,8 @@ #include "mongo/db/query/collation/collator_factory_icu.h" +#include +#include #include "mongo/base/init.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/stdx/memory.h" @@ -827,9 +829,9 @@ TEST(CollatorFactoryICUTest, VersionFieldParsesSuccessfully) { auto collator = factory.makeFromBSON(BSON("locale" << "en_US" << "version" - << "57.1")); + << U_ICU_VERSION)); ASSERT_OK(collator.getStatus()); - ASSERT_EQ("57.1", collator.getValue()->getSpec().version); + ASSERT_EQ(U_ICU_VERSION, collator.getValue()->getSpec().version); } TEST(CollatorFactoryICUTest, VersionFieldPopulatedWhenOmitted) { @@ -837,7 +839,7 @@ TEST(CollatorFactoryICUTest, VersionFieldPopulatedWhenOmitted) { auto collator = factory.makeFromBSON(BSON("locale" << "en_US")); ASSERT_OK(collator.getStatus()); - ASSERT_EQ("57.1", collator.getValue()->getSpec().version); + ASSERT_EQ(U_ICU_VERSION, collator.getValue()->getSpec().version); } TEST(CollatorFactoryICUTest, NonStringVersionFieldFailsToParse) { @@ -1212,12 +1214,24 @@ TEST(CollatorFactoryICUTest, AliasNoForNorwegianBokmalLocaleNotSupported) { .getStatus()); } -TEST(CollatorFactoryICUTest, TraditionalSpanishAliasNotSupported) { - CollatorFactoryICU factory; - ASSERT_NOT_OK(factory - .makeFromBSON(BSON("locale" - << "es__TRADITIONAL")) - .getStatus()); +TEST(CollatorFactoryICUTest, TraditionalSpanishAliasMustMatchICUValidLocale) { + // ICU versions differ in whether this is recognized verbatim or canonicalized to another + // locale. The factory must accept it only when ICU's valid locale matches the request. + const char* localeID = "es__TRADITIONAL"; + const auto locale = icu::Locale::createFromName(localeID); + UErrorCode status = U_ZERO_ERROR; + std::unique_ptr reference(icu::Collator::createInstance(locale, status)); + ASSERT_TRUE(U_SUCCESS(status)); + const auto validLocale = reference->getLocale(ULOC_VALID_LOCALE, status); + ASSERT_TRUE(U_SUCCESS(status)); + CollatorFactoryICU factory; + auto result = factory.makeFromBSON(BSON("locale" << localeID)); + if (StringData(validLocale.getName()) == localeID) { + ASSERT_OK(result.getStatus()); + ASSERT_EQ(result.getValue()->getSpec().localeID, locale.getName()); + } else { + ASSERT_EQ(result.getStatus().code(), ErrorCodes::BadValue); + } } } // namespace diff --git a/src/mongo/db/query/collation/collator_interface_icu.h b/src/mongo/db/query/collation/collator_interface_icu.h index 9bb8382489..232f1d86a6 100644 --- a/src/mongo/db/query/collation/collator_interface_icu.h +++ b/src/mongo/db/query/collation/collator_interface_icu.h @@ -31,10 +31,11 @@ #include "mongo/db/query/collation/collator_interface.h" #include +#include -namespace icu { +U_NAMESPACE_BEGIN class Collator; -} // namespace icu +U_NAMESPACE_END namespace mongo { diff --git a/src/mongo/db/query/collation/collator_interface_icu_test.cpp b/src/mongo/db/query/collation/collator_interface_icu_test.cpp index 821321d793..9cafce8d7c 100644 --- a/src/mongo/db/query/collation/collator_interface_icu_test.cpp +++ b/src/mongo/db/query/collation/collator_interface_icu_test.cpp @@ -33,6 +33,8 @@ #include #include #include +#include +#include #include "mongo/unittest/unittest.h" @@ -92,6 +94,30 @@ void assertEqualEnUS(StringData left, StringData right) { assertEnUSComparison(right, left, ExpectedComparison::EQUAL); } +void assertMalformedUTF8MatchesICU(StringData input) { + // ICU releases differ in how many replacement characters represent an invalid + // sequence. Check the wrapper against this version's UTF-8 decoder, not ICU 57 data. + const auto decoded = icu::UnicodeString::fromUTF8( + icu::StringPiece(input.rawData(), input.size())); + std::string replaced; + decoded.toUTF8String(replaced); + ASSERT_FALSE(replaced.empty()); + ASSERT_NOT_EQUALS(input, StringData(replaced)); + assertEqualEnUS(input, replaced); +} + +std::string referenceSortKey(const icu::Collator& collator, StringData input) { + const auto decoded = icu::UnicodeString::fromUTF8( + icu::StringPiece(input.rawData(), input.size())); + const auto size = collator.getSortKey(decoded, nullptr, 0); + std::vector bytes(size); + ASSERT_EQUALS(size, collator.getSortKey(decoded, bytes.data(), size)); + ASSERT_GREATER_THAN(size, 0); + ASSERT_EQUALS(0, bytes.back()); + // Mongo's ComparisonKey excludes ICU's terminating zero byte. + return std::string(reinterpret_cast(bytes.data()), size - 1); +} + // Returns true if an ICU collator compares 'left' and 'right' as non-equal. Verifies that // the comparison is correct using the compare() function directly and using comparison keys. void assertNotEqualEnUS(StringData left, StringData right) { @@ -438,26 +464,26 @@ TEST(CollatorInterfaceICUTest, InvalidOneByteSeqAndTwoByteSeqCompareEqual) { TEST(CollatorInterfaceICUTest, OverlongASCIICharacterComparesEqualToReplacementCharacter) { // U+002F is the ASCII character "/", which should usually be represented as \x2F. The // representation \xC0\xAF is an unnecessary two-byte encoding of this codepoint. - assertEqualEnUS("\xC0\xAF", u8"\uFFFD"); + assertMalformedUTF8MatchesICU("\xC0\xAF"); } TEST(CollatorInterfaceICUTest, OverlongNullComparesEqualToReplacementCharacter) { // The two-byte sequence \xC0\x80 decodes to U+0000, which should instead be encoded using a // single null byte. - assertEqualEnUS("\xC0\x80", u8"\uFFFD"); + assertMalformedUTF8MatchesICU("\xC0\x80"); } TEST(CollatorInterfaceICUTest, IllegalCodePositionsCompareEqualToReplacementCharacter) { // U+D800 - assertEqualEnUS("\xED\xA0\x80", u8"\uFFFD"); + assertMalformedUTF8MatchesICU("\xED\xA0\x80"); // U+DBFF - assertEqualEnUS("\xED\xAF\xBF", u8"\uFFFD"); + assertMalformedUTF8MatchesICU("\xED\xAF\xBF"); // U+DFFF - assertEqualEnUS("\xED\xBF\xBF", u8"\uFFFD"); + assertMalformedUTF8MatchesICU("\xED\xBF\xBF"); // U+D800, U+DC00 - assertEqualEnUS("\xED\xA0\x80\xED\xB0\x80", u8"\uFFFD\uFFFD"); + assertMalformedUTF8MatchesICU("\xED\xA0\x80\xED\xB0\x80"); // U+DB80, U+DFFF - assertEqualEnUS("\xED\xAE\x80\xED\xBF\xBF", u8"\uFFFD\uFFFD"); + assertMalformedUTF8MatchesICU("\xED\xAE\x80\xED\xBF\xBF"); } TEST(CollatorInterfaceICUTest, UnexpectedTrailingContinuationByteComparesAsReplacementCharacter) { @@ -520,7 +546,7 @@ TEST(CollatorInterfaceICUTest, LastPossibleSequenceOfLengthNotEqualToReplacement TEST(CollatorInterfaceICUTest, CodePointBeyondLargestValidComparesEqualToReplacementCharacter) { // Largest valid code point is U+0010FFFF; U+001FFFFF is higher, and is the last possible valid // four byte sequence. - assertEqualEnUS("\xF7\xBF\xBF\xBF", u8"\uFFFD"); + assertMalformedUTF8MatchesICU("\xF7\xBF\xBF\xBF"); } TEST(CollatorInterfaceICUTest, StringsWithDifferentEmbeddedInvalidSequencesCompareEqual) { @@ -572,11 +598,13 @@ TEST(CollatorInterfaceICUTest, ComparisonKeysForEnUsCollatorCorrect) { std::unique_ptr coll( icu::Collator::createInstance(icu::Locale("en", "US"), status)); ASSERT(U_SUCCESS(status)); + const auto asciiKey = referenceSortKey(*coll, "abc"); + const auto accentedKey = referenceSortKey(*coll, "c\xC3\xB4t\xC3\xA9"); CollatorInterfaceICU icuCollator(collationSpec, std::move(coll)); - ASSERT_EQ(icuCollator.getComparisonKey("abc").getKeyData(), "\x29\x2B\x2D\x01\x07\x01\x07"); + ASSERT_EQ(icuCollator.getComparisonKey("abc").getKeyData(), asciiKey); ASSERT_EQ(icuCollator.getComparisonKey("c\xC3\xB4t\xC3\xA9").getKeyData(), - "\x2D\x45\x4F\x31\x01\x44\x8E\x44\x88\x01\x0A"); + accentedKey); } TEST(CollatorInterfaceICUTest, ComparisonKeysForFrCaCollatorCorrect) { @@ -586,11 +614,13 @@ TEST(CollatorInterfaceICUTest, ComparisonKeysForFrCaCollatorCorrect) { std::unique_ptr coll( icu::Collator::createInstance(icu::Locale("fr", "CA"), status)); ASSERT(U_SUCCESS(status)); + const auto asciiKey = referenceSortKey(*coll, "abc"); + const auto accentedKey = referenceSortKey(*coll, "c\xC3\xB4t\xC3\xA9"); CollatorInterfaceICU icuCollator(collationSpec, std::move(coll)); - ASSERT_EQ(icuCollator.getComparisonKey("abc").getKeyData(), "\x29\x2B\x2D\x01\x07\x01\x07"); + ASSERT_EQ(icuCollator.getComparisonKey("abc").getKeyData(), asciiKey); ASSERT_EQ(icuCollator.getComparisonKey("c\xC3\xB4t\xC3\xA9").getKeyData(), - "\x2D\x45\x4F\x31\x01\x88\x44\x8E\x06\x01\x0A"); + accentedKey); } } // namespace diff --git a/src/mongo/db/query/get_executor_test.cpp b/src/mongo/db/query/get_executor_test.cpp index 002d4a427b..83527ae5d4 100644 --- a/src/mongo/db/query/get_executor_test.cpp +++ b/src/mongo/db/query/get_executor_test.cpp @@ -47,20 +47,18 @@ using namespace mongo; namespace { -using std::unique_ptr; - static const NamespaceString nss("test.collection"); /** * Utility functions to create a CanonicalQuery */ -unique_ptr canonicalize(const char* queryStr, - const char* sortStr, - const char* projStr) { +CanonicalQuery::UPtr canonicalize(const char* queryStr, + const char* sortStr, + const char* projStr) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projStr)); @@ -94,7 +92,7 @@ void testAllowedIndices(std::vector indexes, QuerySettings querySettings; // getAllowedIndices should return false when query shape is not yet in query settings. - unique_ptr cq(canonicalize("{a: 1}", "{}", "{}")); + CanonicalQuery::UPtr cq(canonicalize("{a: 1}", "{}", "{}")); PlanCacheKey key = planCache.computeKey(*cq); ASSERT_FALSE(querySettings.getAllowedIndicesFilter(key)); diff --git a/src/mongo/db/query/plan_cache_test.cpp b/src/mongo/db/query/plan_cache_test.cpp index 30ec089160..b7e37ccd73 100644 --- a/src/mongo/db/query/plan_cache_test.cpp +++ b/src/mongo/db/query/plan_cache_test.cpp @@ -67,11 +67,11 @@ static const NamespaceString nss("test.collection"); /** * Utility functions to create a CanonicalQuery */ -unique_ptr canonicalize(const BSONObj& queryObj) { +CanonicalQuery::UPtr canonicalize(const BSONObj& queryObj) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(queryObj); const boost::intrusive_ptr expCtx; auto statusWithCQ = @@ -84,19 +84,19 @@ unique_ptr canonicalize(const BSONObj& queryObj) { return std::move(statusWithCQ.getValue()); } -unique_ptr canonicalize(const char* queryStr) { +CanonicalQuery::UPtr canonicalize(const char* queryStr) { BSONObj queryObj = fromjson(queryStr); return canonicalize(queryObj); } -unique_ptr canonicalize(const char* queryStr, +CanonicalQuery::UPtr canonicalize(const char* queryStr, const char* sortStr, const char* projStr, const char* collationStr) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projStr)); @@ -112,7 +112,7 @@ unique_ptr canonicalize(const char* queryStr, return std::move(statusWithCQ.getValue()); } -unique_ptr canonicalize(const char* queryStr, +CanonicalQuery::UPtr canonicalize(const char* queryStr, const char* sortStr, const char* projStr, long long skip, @@ -123,7 +123,7 @@ unique_ptr canonicalize(const char* queryStr, QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projStr)); @@ -147,7 +147,7 @@ unique_ptr canonicalize(const char* queryStr, return std::move(statusWithCQ.getValue()); } -unique_ptr canonicalize(const char* queryStr, +CanonicalQuery::UPtr canonicalize(const char* queryStr, const char* sortStr, const char* projStr, long long skip, @@ -159,7 +159,7 @@ unique_ptr canonicalize(const char* queryStr, QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projStr)); @@ -275,12 +275,12 @@ void assertShouldNotCacheQuery(const CanonicalQuery& query) { } void assertShouldNotCacheQuery(const BSONObj& query) { - unique_ptr cq(canonicalize(query)); + CanonicalQuery::UPtr cq(canonicalize(query)); assertShouldNotCacheQuery(*cq); } void assertShouldNotCacheQuery(const char* queryStr) { - unique_ptr cq(canonicalize(queryStr)); + CanonicalQuery::UPtr cq(canonicalize(queryStr)); assertShouldNotCacheQuery(*cq); } @@ -291,12 +291,12 @@ void assertShouldNotCacheQuery(const char* queryStr) { */ TEST(PlanCacheTest, ShouldCacheQueryBasic) { - unique_ptr cq(canonicalize("{a: 1}")); + CanonicalQuery::UPtr cq(canonicalize("{a: 1}")); assertShouldCacheQuery(*cq); } TEST(PlanCacheTest, ShouldCacheQuerySort) { - unique_ptr cq(canonicalize("{}", "{a: -1}", "{_id: 0, a: 1}", "{}")); + CanonicalQuery::UPtr cq(canonicalize("{}", "{a: -1}", "{_id: 0, a: 1}", "{}")); assertShouldCacheQuery(*cq); } @@ -310,7 +310,7 @@ TEST(PlanCacheTest, ShouldCacheQuerySort) { * This should normally be handled by the IDHack runner. */ TEST(PlanCacheTest, ShouldNotCacheQueryCollectionScan) { - unique_ptr cq(canonicalize("{}")); + CanonicalQuery::UPtr cq(canonicalize("{}")); assertShouldNotCacheQuery(*cq); } @@ -320,7 +320,7 @@ TEST(PlanCacheTest, ShouldNotCacheQueryCollectionScan) { * Therefore, not much point in caching. */ TEST(PlanCacheTest, ShouldNotCacheQueryWithHint) { - unique_ptr cq( + CanonicalQuery::UPtr cq( canonicalize("{a: 1}", "{}", "{}", 0, 0, "{a: 1, b: 1}", "{}", "{}")); assertShouldNotCacheQuery(*cq); } @@ -329,7 +329,7 @@ TEST(PlanCacheTest, ShouldNotCacheQueryWithHint) { * Min queries are a specialized case of hinted queries */ TEST(PlanCacheTest, ShouldNotCacheQueryWithMin) { - unique_ptr cq(canonicalize("{a: 1}", "{}", "{}", 0, 0, "{}", "{a: 100}", "{}")); + CanonicalQuery::UPtr cq(canonicalize("{a: 1}", "{}", "{}", 0, 0, "{}", "{a: 100}", "{}")); assertShouldNotCacheQuery(*cq); } @@ -337,7 +337,7 @@ TEST(PlanCacheTest, ShouldNotCacheQueryWithMin) { * Max queries are non-cacheable for the same reasons as min queries. */ TEST(PlanCacheTest, ShouldNotCacheQueryWithMax) { - unique_ptr cq(canonicalize("{a: 1}", "{}", "{}", 0, 0, "{}", "{}", "{a: 100}")); + CanonicalQuery::UPtr cq(canonicalize("{a: 1}", "{}", "{}", 0, 0, "{}", "{}", "{a: 100}")); assertShouldNotCacheQuery(*cq); } @@ -346,7 +346,7 @@ TEST(PlanCacheTest, ShouldNotCacheQueryWithMax) { * the planner is able to come up with a cacheable solution. */ TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinLegacyCoordinates) { - unique_ptr cq( + CanonicalQuery::UPtr cq( canonicalize("{a: {$geoWithin: " "{$box: [[-180, -90], [180, 90]]}}}")); assertShouldCacheQuery(*cq); @@ -356,7 +356,7 @@ TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinLegacyCoordinates) { * $geoWithin queries with GeoJSON coordinates are supported by the index bounds builder. */ TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinJSONCoordinates) { - unique_ptr cq( + CanonicalQuery::UPtr cq( canonicalize("{a: {$geoWithin: " "{$geometry: {type: 'Polygon', coordinates: " "[[[0, 0], [0, 90], [90, 0], [0, 0]]]}}}}")); @@ -367,7 +367,7 @@ TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinJSONCoordinates) { * $geoWithin queries with both legacy and GeoJSON coordinates are cacheable. */ TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinLegacyAndJSONCoordinates) { - unique_ptr cq( + CanonicalQuery::UPtr cq( canonicalize("{$or: [{a: {$geoWithin: {$geometry: {type: 'Polygon', " "coordinates: [[[0, 0], [0, 90], " "[90, 0], [0, 0]]]}}}}," @@ -379,7 +379,7 @@ TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinLegacyAndJSONCoordinates) { * $geoIntersects queries are always cacheable because they support GeoJSON coordinates only. */ TEST(PlanCacheTest, ShouldCacheQueryWithGeoIntersects) { - unique_ptr cq( + CanonicalQuery::UPtr cq( canonicalize("{a: {$geoIntersects: " "{$geometry: {type: 'Point', coordinates: " "[10.0, 10.0]}}}}")); @@ -391,7 +391,7 @@ TEST(PlanCacheTest, ShouldCacheQueryWithGeoIntersects) { * between flat and spherical queries. */ TEST(PlanCacheTest, ShouldNotCacheQueryWithGeoNear) { - unique_ptr cq( + CanonicalQuery::UPtr cq( canonicalize("{a: {$geoNear: {$geometry: {type: 'Point'," "coordinates: [0,0]}, $maxDistance:100}}}")); assertShouldCacheQuery(*cq); @@ -403,7 +403,7 @@ TEST(PlanCacheTest, ShouldNotCacheQueryWithGeoNear) { * non-winning plans. */ TEST(PlanCacheTest, ShouldNotCacheQueryExplain) { - unique_ptr cq(canonicalize("{a: 1}", + CanonicalQuery::UPtr cq(canonicalize("{a: 1}", "{}", "{}", 0, @@ -421,7 +421,7 @@ TEST(PlanCacheTest, ShouldNotCacheQueryExplain) { // Adding an empty vector of query solutions should fail. TEST(PlanCacheTest, AddEmptySolutions) { PlanCache planCache; - unique_ptr cq(canonicalize("{a: 1}")); + CanonicalQuery::UPtr cq(canonicalize("{a: 1}")); std::vector solns; unique_ptr decision(createDecision(1U)); QueryTestServiceContext serviceContext; @@ -430,7 +430,7 @@ TEST(PlanCacheTest, AddEmptySolutions) { TEST(PlanCacheTest, AddValidSolution) { PlanCache planCache; - unique_ptr cq(canonicalize("{a: 1}")); + CanonicalQuery::UPtr cq(canonicalize("{a: 1}")); QuerySolution qs; qs.cacheData.reset(new SolutionCacheData()); qs.cacheData->tree.reset(new PlanCacheIndexTree()); @@ -557,7 +557,7 @@ class CachePlanSelectionTest : public mongo::unittest::Test { // Clean up any previous state from a call to runQueryFull or runQueryAsCommand. solns.clear(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(query); qr->setSort(sort); qr->setProj(proj); @@ -591,7 +591,7 @@ class CachePlanSelectionTest : public mongo::unittest::Test { solns.clear(); const bool isExplain = false; - std::unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); const boost::intrusive_ptr expCtx; @@ -664,7 +664,7 @@ class CachePlanSelectionTest : public mongo::unittest::Test { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(query); qr->setSort(sort); qr->setProj(proj); @@ -677,7 +677,7 @@ class CachePlanSelectionTest : public mongo::unittest::Test { ExtensionsCallbackNoop(), MatchExpressionParser::kAllowAllSpecialFeatures); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr scopedCq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr scopedCq = std::move(statusWithCQ.getValue()); // Create a CachedSolution the long way.. // QuerySolution -> PlanCacheEntry -> CachedSolution @@ -1332,7 +1332,7 @@ void testComputeKey(const char* queryStr, const char* expectedStr) { PlanCache planCache; const char* collationStr = "{}"; - unique_ptr cq(canonicalize(queryStr, sortStr, projStr, collationStr)); + CanonicalQuery::UPtr cq(canonicalize(queryStr, sortStr, projStr, collationStr)); PlanCacheKey key = planCache.computeKey(*cq); PlanCacheKey expectedKey(expectedStr); if (key == expectedKey) { @@ -1412,11 +1412,11 @@ TEST(PlanCacheTest, ComputeKeyGeoWithin) { PlanCache planCache; // Legacy coordinates. - unique_ptr cqLegacy( + CanonicalQuery::UPtr cqLegacy( canonicalize("{a: {$geoWithin: " "{$box: [[-180, -90], [180, 90]]}}}")); // GeoJSON coordinates. - unique_ptr cqNew( + CanonicalQuery::UPtr cqNew( canonicalize("{a: {$geoWithin: " "{$geometry: {type: 'Polygon', coordinates: " "[[[0, 0], [0, 90], [90, 0], [0, 0]]]}}}}")); @@ -1468,9 +1468,9 @@ TEST(PlanCacheTest, ComputeKeySparseIndex) { nullptr, // filterExpr BSONObj())}); - unique_ptr cqEqNumber(canonicalize("{a: 0}}")); - unique_ptr cqEqString(canonicalize("{a: 'x'}}")); - unique_ptr cqEqNull(canonicalize("{a: null}}")); + CanonicalQuery::UPtr cqEqNumber(canonicalize("{a: 0}}")); + CanonicalQuery::UPtr cqEqString(canonicalize("{a: 'x'}}")); + CanonicalQuery::UPtr cqEqNull(canonicalize("{a: null}}")); // 'cqEqNumber' and 'cqEqString' get the same key, since both are compatible with this // index. @@ -1496,9 +1496,9 @@ TEST(PlanCacheTest, ComputeKeyPartialIndex) { filterExpr.get(), BSONObj())}); - unique_ptr cqGtNegativeFive(canonicalize("{f: {$gt: -5}}")); - unique_ptr cqGtZero(canonicalize("{f: {$gt: 0}}")); - unique_ptr cqGtFive(canonicalize("{f: {$gt: 5}}")); + CanonicalQuery::UPtr cqGtNegativeFive(canonicalize("{f: {$gt: -5}}")); + CanonicalQuery::UPtr cqGtZero(canonicalize("{f: {$gt: 0}}")); + CanonicalQuery::UPtr cqGtFive(canonicalize("{f: {$gt: 5}}")); // 'cqGtZero' and 'cqGtFive' get the same key, since both are compatible with this index. ASSERT_EQ(planCache.computeKey(*cqGtZero), planCache.computeKey(*cqGtFive)); @@ -1522,11 +1522,11 @@ TEST(PlanCacheTest, ComputeKeyCollationIndex) { entry.collator = &collator; planCache.notifyOfIndexEntries({entry}); - unique_ptr containsString(canonicalize("{a: 'abc'}")); - unique_ptr containsObject(canonicalize("{a: {b: 'abc'}}")); - unique_ptr containsArray(canonicalize("{a: ['abc', 'xyz']}")); - unique_ptr noStrings(canonicalize("{a: 5}")); - unique_ptr containsStringHasCollation( + CanonicalQuery::UPtr containsString(canonicalize("{a: 'abc'}")); + CanonicalQuery::UPtr containsObject(canonicalize("{a: {b: 'abc'}}")); + CanonicalQuery::UPtr containsArray(canonicalize("{a: ['abc', 'xyz']}")); + CanonicalQuery::UPtr noStrings(canonicalize("{a: 5}")); + CanonicalQuery::UPtr containsStringHasCollation( canonicalize("{a: 'abc'}", "{}", "{}", "{locale: 'mock_reverse_string'}")); // 'containsString', 'containsObject', and 'containsArray' have the same key, since none are @@ -1541,11 +1541,11 @@ TEST(PlanCacheTest, ComputeKeyCollationIndex) { // index. ASSERT_EQ(planCache.computeKey(*noStrings), planCache.computeKey(*containsStringHasCollation)); - unique_ptr inContainsString(canonicalize("{a: {$in: [1, 'abc', 2]}}")); - unique_ptr inContainsObject(canonicalize("{a: {$in: [1, {b: 'abc'}, 2]}}")); - unique_ptr inContainsArray(canonicalize("{a: {$in: [1, ['abc', 'xyz'], 2]}}")); - unique_ptr inNoStrings(canonicalize("{a: {$in: [1, 2]}}")); - unique_ptr inContainsStringHasCollation( + CanonicalQuery::UPtr inContainsString(canonicalize("{a: {$in: [1, 'abc', 2]}}")); + CanonicalQuery::UPtr inContainsObject(canonicalize("{a: {$in: [1, {b: 'abc'}, 2]}}")); + CanonicalQuery::UPtr inContainsArray(canonicalize("{a: {$in: [1, ['abc', 'xyz'], 2]}}")); + CanonicalQuery::UPtr inNoStrings(canonicalize("{a: {$in: [1, 2]}}")); + CanonicalQuery::UPtr inContainsStringHasCollation( canonicalize("{a: {$in: [1, 'abc', 2]}}", "{}", "{}", "{locale: 'mock_reverse_string'}")); // 'inContainsString', 'inContainsObject', and 'inContainsArray' have the same key, since none diff --git a/src/mongo/db/query/query_planner_test.cpp b/src/mongo/db/query/query_planner_test.cpp index 0a6a2d3aed..8d941b5fb5 100644 --- a/src/mongo/db/query/query_planner_test.cpp +++ b/src/mongo/db/query/query_planner_test.cpp @@ -4313,11 +4313,11 @@ TEST_F(QueryPlannerTest, CacheDataFromTaggedTreeFailsOnBadInput) { // No relevant index matching the index tag. relevantIndices.push_back(IndexEntry(BSON("a" << 1))); - auto qr = stdx::make_unique(NamespaceString("test.collection")); + auto qr = ObjectPool::newObject(NamespaceString("test.collection")); qr->setFilter(BSON("a" << 3)); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - std::unique_ptr scopedCq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr scopedCq = std::move(statusWithCQ.getValue()); scopedCq->root()->setTag(new IndexTag(1)); ASSERT_NOT_OK( @@ -4327,11 +4327,11 @@ TEST_F(QueryPlannerTest, CacheDataFromTaggedTreeFailsOnBadInput) { TEST_F(QueryPlannerTest, TagAccordingToCacheFailsOnBadInput) { const NamespaceString nss("test.collection"); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(BSON("a" << 3)); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - std::unique_ptr scopedCq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr scopedCq = std::move(statusWithCQ.getValue()); std::unique_ptr indexTree(new PlanCacheIndexTree()); indexTree->setIndexEntry(IndexEntry(BSON("a" << 1), "a_1")); @@ -4356,7 +4356,7 @@ TEST_F(QueryPlannerTest, TagAccordingToCacheFailsOnBadInput) { ASSERT_OK(s); // Regenerate canonical query in order to clear tags. - auto newQR = stdx::make_unique(nss); + auto newQR = ObjectPool::newObject(nss); newQR->setFilter(BSON("a" << 3)); statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(newQR)); ASSERT_OK(statusWithCQ.getStatus()); diff --git a/src/mongo/db/query/query_planner_test_fixture.cpp b/src/mongo/db/query/query_planner_test_fixture.cpp index 63ec2465f5..c457fe5dd1 100644 --- a/src/mongo/db/query/query_planner_test_fixture.cpp +++ b/src/mongo/db/query/query_planner_test_fixture.cpp @@ -225,7 +225,7 @@ void QueryPlannerTest::runQueryFull(const BSONObj& query, solns.clear(); cq.reset(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(query); qr->setSort(sort); qr->setProj(proj); @@ -307,7 +307,7 @@ void QueryPlannerTest::runInvalidQueryFull(const BSONObj& query, solns.clear(); cq.reset(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(query); qr->setSort(sort); qr->setProj(proj); @@ -346,7 +346,7 @@ void QueryPlannerTest::runQueryAsCommand(const BSONObj& cmdObj) { invariant(nss.isValid()); const bool isExplain = false; - std::unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); const boost::intrusive_ptr expCtx; @@ -371,7 +371,7 @@ void QueryPlannerTest::runInvalidQueryAsCommand(const BSONObj& cmdObj) { invariant(nss.isValid()); const bool isExplain = false; - std::unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); const boost::intrusive_ptr expCtx; diff --git a/src/mongo/db/query/query_planner_test_fixture.h b/src/mongo/db/query/query_planner_test_fixture.h index 2bdd7d5b16..6a9e15ed52 100644 --- a/src/mongo/db/query/query_planner_test_fixture.h +++ b/src/mongo/db/query/query_planner_test_fixture.h @@ -206,7 +206,7 @@ class QueryPlannerTest : public mongo::unittest::Test { QueryTestServiceContext serviceContext; ServiceContext::UniqueOperationContext opCtx; BSONObj queryObj; - std::unique_ptr cq; + CanonicalQuery::UPtr cq; QueryPlannerParams params; std::vector> solns; }; diff --git a/src/mongo/db/query/query_request_test.cpp b/src/mongo/db/query/query_request_test.cpp index c9354dfe1f..ae905d8e49 100644 --- a/src/mongo/db/query/query_request_test.cpp +++ b/src/mongo/db/query/query_request_test.cpp @@ -40,6 +40,7 @@ #include "mongo/db/pipeline/aggregation_request.h" #include "mongo/db/query/query_request.h" #include "mongo/db/service_context_test_fixture.h" +#include "mongo/db/storage/recovery_unit_noop.h" #include "mongo/unittest/unittest.h" namespace mongo { @@ -369,7 +370,7 @@ TEST(QueryRequestTest, ParseFromCommandWithOptions) { "maxScan: 1000}}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); // Make sure the values from the command BSON are reflected in the QR. @@ -384,7 +385,7 @@ TEST(QueryRequestTest, ParseFromCommandHintAsString) { "hint: 'foo_1'}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); BSONObj hintObj = qr->getHint(); @@ -423,7 +424,7 @@ TEST(QueryRequestTest, ParseFromCommandAllFlagsTrue) { "allowPartialResults: true}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); // Test that all the flags got set to true. @@ -443,7 +444,7 @@ TEST(QueryRequestTest, ParseFromCommandCommentWithValidMinMax) { "max: {a: 2}}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_EQUALS("the comment", qr->getComment()); @@ -469,7 +470,7 @@ TEST(QueryRequestTest, ParseFromCommandAllNonOptionFields) { "singleBatch: false}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); // Check the values inside the QR. @@ -501,7 +502,7 @@ TEST(QueryRequestTest, ParseFromCommandLargeLimit) { "limit: 8000000000}"); // 8 * 1000 * 1000 * 1000 const NamespaceString nss("test.testns"); const bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_EQUALS(8LL * 1000 * 1000 * 1000, *qr->getLimit()); @@ -514,7 +515,7 @@ TEST(QueryRequestTest, ParseFromCommandLargeBatchSize) { "batchSize: 8000000000}"); // 8 * 1000 * 1000 * 1000 const NamespaceString nss("test.testns"); const bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_EQUALS(8LL * 1000 * 1000 * 1000, *qr->getBatchSize()); @@ -527,7 +528,7 @@ TEST(QueryRequestTest, ParseFromCommandLargeSkip) { "skip: 8000000000}"); // 8 * 1000 * 1000 * 1000 const NamespaceString nss("test.testns"); const bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_EQUALS(8LL * 1000 * 1000 * 1000, *qr->getSkip()); @@ -817,7 +818,7 @@ TEST(QueryRequestTest, ParseFromCommandSkipIsZero) { "filter: {a: 3}}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_BSONOBJ_EQ(BSON("a" << 3), qr->getFilter()); ASSERT_FALSE(qr->getSkip()); @@ -841,7 +842,7 @@ TEST(QueryRequestTest, ParseFromCommandLimitIsZero) { "filter: {a: 3}}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_BSONOBJ_EQ(BSON("a" << 3), qr->getFilter()); ASSERT_FALSE(qr->getLimit()); @@ -862,7 +863,7 @@ TEST(QueryRequestTest, ParseFromCommandBatchSizeZero) { BSONObj cmdObj = fromjson("{find: 'testns', batchSize: 0}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT(qr->getBatchSize()); @@ -875,7 +876,7 @@ TEST(QueryRequestTest, ParseFromCommandDefaultBatchSize) { BSONObj cmdObj = fromjson("{find: 'testns'}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT(!qr->getBatchSize()); @@ -948,7 +949,7 @@ TEST(QueryRequestTest, ParseCommandIsFromFindCommand) { BSONObj cmdObj = fromjson("{find: 'testns'}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_FALSE(qr->getNToReturn()); @@ -982,7 +983,7 @@ TEST(QueryRequestTest, DefaultQueryParametersCorrect) { BSONObj cmdObj = fromjson("{find: 'testns'}"); const NamespaceString nss("test.testns"); - std::unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, false))); ASSERT_FALSE(qr->getSkip()); @@ -1281,7 +1282,7 @@ TEST(QueryRequestTest, ParseFromLegacyObjMetaOpComment) { "{$query: {a: 1}," "$comment: {b: 2, c: {d: 'ParseFromLegacyObjMetaOpComment'}}}"); const NamespaceString nss("test.testns"); - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::fromLegacyQuery(nss, queryObj, BSONObj(), 0, 0, 0))); // Ensure that legacy comment meta-operator is parsed to a string comment @@ -1294,7 +1295,7 @@ TEST(QueryRequestTest, ParseFromLegacyStringMetaOpComment) { "{$query: {a: 1}," "$comment: 'ParseFromLegacyStringMetaOpComment'}"); const NamespaceString nss("test.testns"); - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::fromLegacyQuery(nss, queryObj, BSONObj(), 0, 0, 0))); ASSERT_EQ(qr->getComment(), "ParseFromLegacyStringMetaOpComment"); @@ -1315,7 +1316,7 @@ TEST(QueryRequestTest, ParseFromLegacyQuery) { $maxScan: 7 })"); const NamespaceString nss("test.testns"); - unique_ptr qr(assertGet(QueryRequest::fromLegacyQuery( + QueryRequest::UPtr qr(assertGet(QueryRequest::fromLegacyQuery( nss, queryObj, BSON("proj" << 1), kSkip, kNToReturn, QueryOption_Exhaust))); ASSERT_EQ(qr->nss(), nss); @@ -1344,7 +1345,7 @@ TEST(QueryRequestTest, ParseFromLegacyQueryUnwrapped) { foo: 1 })"); const NamespaceString nss("test.testns"); - unique_ptr qr(assertGet( + QueryRequest::UPtr qr(assertGet( QueryRequest::fromLegacyQuery(nss, queryObj, BSONObj(), 0, 0, QueryOption_Exhaust))); ASSERT_EQ(qr->nss(), nss); @@ -1367,12 +1368,15 @@ class QueryRequestTest : public ServiceContextTest {}; TEST_F(QueryRequestTest, ParseFromUUID) { auto opCtx = makeOperationContext(); + // UUIDCatalog registers recovery-unit changes; this parser test needs no real engine. + opCtx->setRecoveryUnit(new RecoveryUnitNoop(), + WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); // Register a UUID/Collection pair in the UUIDCatalog. const CollectionUUID uuid = UUID::gen(); const NamespaceString nss("test.testns"); - Collection coll(stdx::make_unique(nss)); + auto coll = stdx::make_unique(stdx::make_unique(nss)); UUIDCatalog& catalog = UUIDCatalog::get(opCtx.get()); - catalog.onCreateCollection(opCtx.get(), &coll, uuid); + catalog.onCreateCollection(opCtx.get(), std::move(coll), uuid); QueryRequest qr(uuid); // Ensure a call to refreshNSS succeeds. qr.refreshNSS(opCtx.get()); diff --git a/src/mongo/db/record_id.h b/src/mongo/db/record_id.h index fbd81241e1..421317ecd3 100644 --- a/src/mongo/db/record_id.h +++ b/src/mongo/db/record_id.h @@ -227,10 +227,9 @@ class RecordId { // repr() now just for Format::kLong int64_t repr() const { - // In the Eloq engine, the RecordId is equivalent to the "_id" field and is stored as a - // string. Therefore, the repr() function is meaningless. - // Returning 0 ensures compatibility with the existing API. - return 0; + // Preserve the legacy integer API for integer-backed stores and sentinels. + // Eloq's string-backed IDs have no integer representation. + return isLong() ? _getLongNoCheck() : 0; } /** diff --git a/src/mongo/db/record_id_test.cpp b/src/mongo/db/record_id_test.cpp index 389549bccc..965dcf094a 100644 --- a/src/mongo/db/record_id_test.cpp +++ b/src/mongo/db/record_id_test.cpp @@ -35,6 +35,18 @@ namespace mongo { namespace { +TEST(RecordId, IntegerRepresentationAndNullSentinel) { + ASSERT_EQUALS(42, RecordId(42).repr()); + ASSERT_EQUALS(-7, RecordId(-7).repr()); + ASSERT_FALSE(RecordId(42).isNull()); + ASSERT_TRUE(RecordId(0).isNull()); + ASSERT_TRUE(RecordId().isNull()); + RecordId stringId("key", 3); + ASSERT_TRUE(stringId.isStr()); + ASSERT_FALSE(stringId.isNull()); + ASSERT_EQUALS("key", stringId.getStringView()); +} + TEST(RecordId, HashEqual) { RecordId locA(1, 2); RecordId locB; diff --git a/src/mongo/db/repl/replication_coordinator_standalone.cpp b/src/mongo/db/repl/replication_coordinator_standalone.cpp new file mode 100644 index 0000000000..3a89d59274 --- /dev/null +++ b/src/mongo/db/repl/replication_coordinator_standalone.cpp @@ -0,0 +1,485 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kReplication + +#include "mongo/platform/basic.h" + +#include "mongo/db/repl/replication_coordinator_standalone.h" + +#include "mongo/db/global_settings.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/repl/read_concern_args.h" +#include "mongo/db/repl/repl_set_config.h" +#include "mongo/db/storage/storage_engine.h" +#include "mongo/util/assert_util.h" + +namespace mongo { +namespace repl { + +namespace { + +// Only replication-only APIs without a Status result use this helper. APIs used by +// ordinary standalone commands must preserve their non-throwing Status contract. +MONGO_COMPILER_NORETURN void replicationUnavailable() { + uasserted(ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"); +} + +} // namespace + +ReplicationCoordinatorStandalone::ReplicationCoordinatorStandalone(ServiceContext* service) + : _service(service) {} + +ReplicationCoordinatorStandalone::~ReplicationCoordinatorStandalone() = default; + +void ReplicationCoordinatorStandalone::startup(OperationContext* opCtx) {} + +void ReplicationCoordinatorStandalone::shutdown(OperationContext* opCtx) {} + +const ReplSettings& ReplicationCoordinatorStandalone::getSettings() const { + return getGlobalReplSettings(); +} + +ReplicationCoordinator::Mode ReplicationCoordinatorStandalone::getReplicationMode() const { + return ReplicationCoordinator::Mode::modeNone; +} + +bool ReplicationCoordinatorStandalone::isMasterForReportingPurposes() { + return true; +} + +bool ReplicationCoordinatorStandalone::canAcceptWritesForDatabase(OperationContext* opCtx, + StringData dbName) { + return true; +} + +bool ReplicationCoordinatorStandalone::canAcceptWritesForDatabase_UNSAFE(OperationContext* opCtx, + StringData dbName) { + return true; +} + +bool ReplicationCoordinatorStandalone::canAcceptWritesFor(OperationContext* opCtx, + const NamespaceString& ns) { + return true; +} + +bool ReplicationCoordinatorStandalone::canAcceptWritesFor_UNSAFE(OperationContext* opCtx, + const NamespaceString& ns) { + return true; +} + +Status ReplicationCoordinatorStandalone::checkCanServeReadsFor(OperationContext* opCtx, + const NamespaceString& ns, + bool slaveOk) { + return Status::OK(); +} + +Status ReplicationCoordinatorStandalone::checkCanServeReadsFor_UNSAFE(OperationContext* opCtx, + const NamespaceString& ns, + bool slaveOk) { + return Status::OK(); +} + +bool ReplicationCoordinatorStandalone::isInPrimaryOrSecondaryState() const { + return false; +} + +bool ReplicationCoordinatorStandalone::shouldRelaxIndexConstraints(OperationContext* opCtx, + const NamespaceString& ns) { + return false; +} + +bool ReplicationCoordinatorStandalone::getMaintenanceMode() { + return false; +} + +WriteConcernOptions ReplicationCoordinatorStandalone::getGetLastErrorDefault() { + return WriteConcernOptions(); +} + +bool ReplicationCoordinatorStandalone::isReplEnabled() const { + return false; +} + +WriteConcernOptions ReplicationCoordinatorStandalone::populateUnsetWriteConcernOptionsSyncMode( + WriteConcernOptions wc) { + WriteConcernOptions writeConcern(wc); + if (writeConcern.syncMode == WriteConcernOptions::SyncMode::UNSET) { + writeConcern.syncMode = WriteConcernOptions::SyncMode::NONE; + } + return writeConcern; +} + +bool ReplicationCoordinatorStandalone::buildsIndexes() { + return true; +} + +OpTime ReplicationCoordinatorStandalone::getCurrentCommittedSnapshotOpTime() const { + return OpTime(); +} + +void ReplicationCoordinatorStandalone::appendDiagnosticBSON(mongo::BSONObjBuilder*) {} + +void ReplicationCoordinatorStandalone::appendConnectionStats( + executor::ConnectionPoolStats* stats) const {} + +MemberState ReplicationCoordinatorStandalone::getMemberState() const { + return MemberState::RS_STARTUP; +} + +Status ReplicationCoordinatorStandalone::waitForMemberState(MemberState, Milliseconds) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Seconds ReplicationCoordinatorStandalone::getSlaveDelaySecs() const { + return Seconds(0); +} + +void ReplicationCoordinatorStandalone::clearSyncSourceBlacklist() {} + +Status ReplicationCoordinatorStandalone::setFollowerMode(const MemberState&) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +ReplicationCoordinator::ApplierState ReplicationCoordinatorStandalone::getApplierState() { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::signalDrainComplete(OperationContext*, long long) { + replicationUnavailable(); +} + +Status ReplicationCoordinatorStandalone::waitForDrainFinish(Milliseconds) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +void ReplicationCoordinatorStandalone::signalUpstreamUpdater() { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::setMyHeartbeatMessage(const std::string&) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::setMyLastAppliedOpTimeForward(const OpTime&, + DataConsistency) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::setMyLastDurableOpTimeForward(const OpTime&) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::setMyLastAppliedOpTime(const OpTime&) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::setMyLastDurableOpTime(const OpTime&) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::resetMyLastOpTimes() { + replicationUnavailable(); +} + +OpTime ReplicationCoordinatorStandalone::getMyLastAppliedOpTime() const { + return OpTime(); +} + +OpTime ReplicationCoordinatorStandalone::getMyLastDurableOpTime() const { + return OpTime(); +} + +Status ReplicationCoordinatorStandalone::waitUntilOpTimeForRead( + OperationContext* opCtx, const ReadConcernArgs& readConcern) { + // Preserve the normal modeNone validation; storage capabilities belong to the engine. + const auto level = readConcern.getLevel(); + if (readConcern.getArgsAfterClusterTime() && level != ReadConcernLevel::kMajorityReadConcern && + level != ReadConcernLevel::kLocalReadConcern && + level != ReadConcernLevel::kSnapshotReadConcern) { + return {ErrorCodes::BadValue, + "Only readConcern level 'majority', 'local', or 'snapshot' is allowed when " + "specifying afterClusterTime"}; + } + if (readConcern.getArgsAtClusterTime() && level != ReadConcernLevel::kSnapshotReadConcern) { + return {ErrorCodes::BadValue, + "readConcern level 'snapshot' is required when specifying atClusterTime"}; + } + if (level == ReadConcernLevel::kMajorityReadConcern && + !opCtx->getServiceContext()->getStorageEngine()->supportsReadConcernMajority()) { + return {ErrorCodes::ReadConcernMajorityNotEnabled, + "Storage engine does not support majority read concern"}; + } + if (level == ReadConcernLevel::kSnapshotReadConcern && + !opCtx->getServiceContext()->getStorageEngine()->supportsReadConcernSnapshot()) { + return {ErrorCodes::InvalidOptions, + "Storage engine does not support snapshot read concern"}; + } + if (!readConcern.getArgsAfterClusterTime() && !readConcern.getArgsOpTime() && + !readConcern.getArgsAtClusterTime()) { + return Status::OK(); + } + return waitUntilOpTimeForReadUntil(opCtx, readConcern, boost::none); +} + +Status ReplicationCoordinatorStandalone::waitUntilOpTimeForReadUntil(OperationContext*, + const ReadConcernArgs&, + boost::optional) { + return {ErrorCodes::NotAReplicaSet, + "node needs to be a replica set member to use timestamped read concern"}; +} + +ReplicationCoordinator::StatusAndDuration ReplicationCoordinatorStandalone::awaitReplication( + OperationContext*, const OpTime&, const WriteConcernOptions&) { + return {Status::OK(), Milliseconds(0)}; +} + +Status ReplicationCoordinatorStandalone::stepDown(OperationContext*, + const bool, + const Milliseconds&, + const Milliseconds&) { + // The normal shutdown command always tries stepDown and explicitly accepts NotMaster. + return {ErrorCodes::NotMaster, "not running as a replica-set primary"}; +} + +OID ReplicationCoordinatorStandalone::getElectionId() { + return OID(); +} + +int ReplicationCoordinatorStandalone::getMyId() const { + return -1; +} + +Status ReplicationCoordinatorStandalone::resyncData(OperationContext*, bool) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +StatusWith ReplicationCoordinatorStandalone::prepareReplSetUpdatePositionCommand() const { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetGetStatus(BSONObjBuilder*, + ReplSetGetStatusResponseStyle) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +void ReplicationCoordinatorStandalone::fillIsMasterForReplSet(IsMasterResponse*) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::appendSlaveInfoData(BSONObjBuilder*) {} + +ReplSetConfig ReplicationCoordinatorStandalone::getConfig() const { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::processReplSetGetConfig(BSONObjBuilder*) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::processReplSetMetadata(const rpc::ReplSetMetadata&) {} + +void ReplicationCoordinatorStandalone::cancelAndRescheduleElectionTimeout() {} + +Status ReplicationCoordinatorStandalone::setMaintenanceMode(bool) { + // MaintenanceModeSetter tolerates a failed Status, but not an exception. + return {ErrorCodes::NoReplicationEnabled, + "can only set maintenance mode on replica set members"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetSyncFrom(OperationContext*, + const HostAndPort&, + BSONObjBuilder*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetFreeze(int, BSONObjBuilder*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processHeartbeat(const ReplSetHeartbeatArgs&, + ReplSetHeartbeatResponse*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetReconfig(OperationContext*, + const ReplSetReconfigArgs&, + BSONObjBuilder*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetInitiate(OperationContext*, + const BSONObj&, + BSONObjBuilder*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::abortCatchupIfNeeded() { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetFresh(const ReplSetFreshArgs&, + BSONObjBuilder*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetElect(const ReplSetElectArgs&, + BSONObjBuilder*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetUpdatePosition(const UpdatePositionArgs&, + long long*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +std::vector ReplicationCoordinatorStandalone::getHostsWrittenTo(const OpTime&, bool) { + return {}; +} + +std::vector ReplicationCoordinatorStandalone::getOtherNodesInReplSet() const { + return {}; +} + +Status ReplicationCoordinatorStandalone::checkIfWriteConcernCanBeSatisfied( + const WriteConcernOptions&) const { + return {ErrorCodes::NoReplicationEnabled, "not running with --replSet"}; +} + +Status ReplicationCoordinatorStandalone::checkReplEnabledForCommand(BSONObjBuilder*) { + return Status(ErrorCodes::NoReplicationEnabled, "not running with --replSet"); +} + +HostAndPort ReplicationCoordinatorStandalone::chooseNewSyncSource(const OpTime&) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::blacklistSyncSource(const HostAndPort&, Date_t) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::resetLastOpTimesFromOplog(OperationContext*, + DataConsistency) { + replicationUnavailable(); +} + +bool ReplicationCoordinatorStandalone::shouldChangeSyncSource( + const HostAndPort&, const rpc::ReplSetMetadata&, boost::optional) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::advanceCommitPoint(const OpTime&) { + replicationUnavailable(); +} + +OpTime ReplicationCoordinatorStandalone::getLastCommittedOpTime() const { + return OpTime(); +} + +Status ReplicationCoordinatorStandalone::processReplSetRequestVotes(OperationContext*, + const ReplSetRequestVotesArgs&, + ReplSetRequestVotesResponse*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +void ReplicationCoordinatorStandalone::prepareReplMetadata(const BSONObj&, + const OpTime&, + BSONObjBuilder*) const {} + +bool ReplicationCoordinatorStandalone::isV1ElectionProtocol() const { + return false; +} + +bool ReplicationCoordinatorStandalone::getWriteConcernMajorityShouldJournal() { + return false; +} + +Status ReplicationCoordinatorStandalone::processHeartbeatV1(const ReplSetHeartbeatArgsV1&, + ReplSetHeartbeatResponse*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +void ReplicationCoordinatorStandalone::summarizeAsHtml(ReplSetHtmlSummary*) { + replicationUnavailable(); +} + +long long ReplicationCoordinatorStandalone::getTerm() { + return OpTime::kUninitializedTerm; +} + +Status ReplicationCoordinatorStandalone::updateTerm(OperationContext*, long long) { + return {ErrorCodes::BadValue, "cannot supply 'term' without active replication"}; +} + +void ReplicationCoordinatorStandalone::waitUntilSnapshotCommitted(OperationContext*, + const Timestamp&) { + replicationUnavailable(); +} + +size_t ReplicationCoordinatorStandalone::getNumUncommittedSnapshots() { + return 0; +} + +void ReplicationCoordinatorStandalone::dropAllSnapshots() {} + +Status ReplicationCoordinatorStandalone::stepUpIfEligible(bool skipDryRun) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +ReplSettings::IndexPrefetchConfig ReplicationCoordinatorStandalone::getIndexPrefetchConfig() const { + return _indexPrefetchConfig.load(); +} + +void ReplicationCoordinatorStandalone::setIndexPrefetchConfig( + const ReplSettings::IndexPrefetchConfig config) { + _indexPrefetchConfig.store(config); +} + +void ReplicationCoordinatorStandalone::signalDropPendingCollectionsRemovedFromStorage() {} + +} // namespace repl +} // namespace mongo diff --git a/src/mongo/db/repl/replication_coordinator_standalone.h b/src/mongo/db/repl/replication_coordinator_standalone.h new file mode 100644 index 0000000000..adf3f50133 --- /dev/null +++ b/src/mongo/db/repl/replication_coordinator_standalone.h @@ -0,0 +1,253 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/repl/replication_coordinator.h" + +namespace mongo { +namespace repl { + +/** + * MongoDB standalone coordination only. Distributed transactions, replication and durability + * belong to the Eloq storage engine; this class must never start MongoDB replica-set workers. + */ +class ReplicationCoordinatorStandalone final : public repl::ReplicationCoordinator { + +public: + explicit ReplicationCoordinatorStandalone(ServiceContext* serviceContext); + ~ReplicationCoordinatorStandalone() override; + + ReplicationCoordinatorStandalone(const ReplicationCoordinatorStandalone&) = delete; + ReplicationCoordinatorStandalone& operator=(const ReplicationCoordinatorStandalone&) = delete; + + // Standalone implementations of the shared ReplicationCoordinator interface. + + void startup(OperationContext* opCtx) override; + + void shutdown(OperationContext* opCtx) override; + + // Returns the ServiceContext where this instance runs. + ServiceContext* getServiceContext() override { + return _service; + } + + const repl::ReplSettings& getSettings() const override; + + Mode getReplicationMode() const override; + bool getMaintenanceMode() override; + + bool isReplEnabled() const override; + bool isMasterForReportingPurposes() override; + bool isInPrimaryOrSecondaryState() const override; + + bool canAcceptWritesForDatabase(OperationContext* opCtx, StringData dbName) override; + bool canAcceptWritesForDatabase_UNSAFE(OperationContext* opCtx, StringData dbName) override; + + bool canAcceptWritesFor(OperationContext* opCtx, const NamespaceString& ns) override; + bool canAcceptWritesFor_UNSAFE(OperationContext* opCtx, const NamespaceString& ns) override; + + Status checkCanServeReadsFor(OperationContext* opCtx, + const NamespaceString& ns, + bool slaveOk) override; + Status checkCanServeReadsFor_UNSAFE(OperationContext* opCtx, + const NamespaceString& ns, + bool slaveOk) override; + + bool shouldRelaxIndexConstraints(OperationContext* opCtx, const NamespaceString& ns) override; + + WriteConcernOptions getGetLastErrorDefault() override; + + WriteConcernOptions populateUnsetWriteConcernOptionsSyncMode(WriteConcernOptions wc) override; + + bool buildsIndexes() override; + + // Replica-set operations reject requests; local server operations have modeNone semantics. + + + repl::MemberState getMemberState() const override; + + Status waitForMemberState(repl::MemberState, Milliseconds) override; + + Seconds getSlaveDelaySecs() const override; + + void clearSyncSourceBlacklist() override; + + repl::ReplicationCoordinator::StatusAndDuration awaitReplication( + OperationContext*, const repl::OpTime&, const WriteConcernOptions&) override; + + Status stepDown(OperationContext*, bool, const Milliseconds&, const Milliseconds&) override; + + Status checkIfWriteConcernCanBeSatisfied(const WriteConcernOptions&) const override; + + void setMyLastAppliedOpTime(const repl::OpTime&) override; + void setMyLastDurableOpTime(const repl::OpTime&) override; + + void setMyLastAppliedOpTimeForward(const repl::OpTime&, DataConsistency) override; + void setMyLastDurableOpTimeForward(const repl::OpTime&) override; + + void resetMyLastOpTimes() override; + + void setMyHeartbeatMessage(const std::string&) override; + + repl::OpTime getMyLastAppliedOpTime() const override; + repl::OpTime getMyLastDurableOpTime() const override; + + Status waitUntilOpTimeForReadUntil(OperationContext*, + const repl::ReadConcernArgs&, + boost::optional) override; + + Status waitUntilOpTimeForRead(OperationContext*, const repl::ReadConcernArgs&) override; + + OID getElectionId() override; + + int getMyId() const override; + + Status setFollowerMode(const repl::MemberState&) override; + + ApplierState getApplierState() override; + + void signalDrainComplete(OperationContext*, long long) override; + + Status waitForDrainFinish(Milliseconds) override; + + void signalUpstreamUpdater() override; + + Status resyncData(OperationContext*, bool) override; + + StatusWith prepareReplSetUpdatePositionCommand() const override; + + Status processReplSetGetStatus(BSONObjBuilder*, ReplSetGetStatusResponseStyle) override; + + void fillIsMasterForReplSet(repl::IsMasterResponse*) override; + + void appendSlaveInfoData(BSONObjBuilder*) override; + + repl::ReplSetConfig getConfig() const override; + + void processReplSetGetConfig(BSONObjBuilder*) override; + + void processReplSetMetadata(const rpc::ReplSetMetadata&) override; + + void advanceCommitPoint(const repl::OpTime& committedOpTime) override; + + void cancelAndRescheduleElectionTimeout() override; + + Status setMaintenanceMode(bool) override; + + Status processReplSetSyncFrom(OperationContext*, const HostAndPort&, BSONObjBuilder*) override; + + Status processReplSetFreeze(int, BSONObjBuilder*) override; + + Status processHeartbeat(const repl::ReplSetHeartbeatArgs&, + repl::ReplSetHeartbeatResponse*) override; + + Status processReplSetReconfig(OperationContext*, + const ReplSetReconfigArgs&, + BSONObjBuilder*) override; + + Status processReplSetInitiate(OperationContext*, const BSONObj&, BSONObjBuilder*) override; + + Status processReplSetFresh(const ReplSetFreshArgs&, BSONObjBuilder*) override; + + Status processReplSetElect(const ReplSetElectArgs& args, BSONObjBuilder* response) override; + + Status processReplSetUpdatePosition(const repl::UpdatePositionArgs&, long long*) override; + + std::vector getHostsWrittenTo(const repl::OpTime&, bool) override; + + std::vector getOtherNodesInReplSet() const override; + + Status checkReplEnabledForCommand(BSONObjBuilder*) override; + + + HostAndPort chooseNewSyncSource(const repl::OpTime&) override; + + void blacklistSyncSource(const HostAndPort&, Date_t) override; + + void resetLastOpTimesFromOplog(OperationContext*, DataConsistency) override; + + bool shouldChangeSyncSource(const HostAndPort&, + const rpc::ReplSetMetadata&, + boost::optional) override; + + repl::OpTime getLastCommittedOpTime() const override; + + Status processReplSetRequestVotes(OperationContext*, + const repl::ReplSetRequestVotesArgs&, + repl::ReplSetRequestVotesResponse*) override; + + void prepareReplMetadata(const BSONObj&, const repl::OpTime&, BSONObjBuilder*) const override; + + Status processHeartbeatV1(const repl::ReplSetHeartbeatArgsV1&, + repl::ReplSetHeartbeatResponse*) override; + + bool isV1ElectionProtocol() const override; + + bool getWriteConcernMajorityShouldJournal() override; + + void summarizeAsHtml(repl::ReplSetHtmlSummary*) override; + + void dropAllSnapshots() override; + + long long getTerm() override; + + Status updateTerm(OperationContext*, long long) override; + + repl::OpTime getCurrentCommittedSnapshotOpTime() const override; + + void waitUntilSnapshotCommitted(OperationContext*, const Timestamp&) override; + + void appendDiagnosticBSON(BSONObjBuilder*) override; + + void appendConnectionStats(executor::ConnectionPoolStats* stats) const override; + + size_t getNumUncommittedSnapshots() override; + + repl::ReplSettings::IndexPrefetchConfig getIndexPrefetchConfig() const override; + void setIndexPrefetchConfig(const repl::ReplSettings::IndexPrefetchConfig) override; + + Status stepUpIfEligible(bool skipDryRun) override; + + Status abortCatchupIfNeeded() override; + + void signalDropPendingCollectionsRemovedFromStorage() final; + +private: + // Back pointer to the ServiceContext that has started the instance. + ServiceContext* const _service; + + // Preserve the server parameter's set/get behavior without starting a prefetch worker. + std::atomic _indexPrefetchConfig{ + ReplSettings::IndexPrefetchConfig::PREFETCH_ALL}; +}; + +} // namespace repl +} // namespace mongo diff --git a/src/mongo/db/repl/replication_info.cpp b/src/mongo/db/repl/replication_info.cpp index ca56b59443..59b3bcde03 100644 --- a/src/mongo/db/repl/replication_info.cpp +++ b/src/mongo/db/repl/replication_info.cpp @@ -125,6 +125,8 @@ void appendReplicationInfo(OperationContext* opCtx, BSONObjBuilder& result, int t.done(); } +#ifndef ELOQDOC_STANDALONE + // Only the full server links the legacy remote replication-source client. if (level > 1) { invariant(!opCtx->lockState()->isLocked()); // note: there is no so-style timeout on this connection; perhaps we should have @@ -144,6 +146,7 @@ void appendReplicationInfo(OperationContext* opCtx, BSONObjBuilder& result, int } conn.done(); } +#endif sources.append(BSONObjBuilder::numStr(n++), bb.obj()); } diff --git a/src/mongo/db/server_options_server_helpers.cpp b/src/mongo/db/server_options_server_helpers.cpp index 464920f802..67f5d68c6e 100644 --- a/src/mongo/db/server_options_server_helpers.cpp +++ b/src/mongo/db/server_options_server_helpers.cpp @@ -133,8 +133,7 @@ Status addGeneralServerOptions(moe::OptionSection* options) { ->addOptionChaining("net.adaptiveThreadNum", "adaptiveThreadNum", moe::Int, - "set the thread num for adaptive service executor mode") - .setDefault(moe::Value(1)); + "set the thread num for adaptive service executor mode"); #if MONGO_ENTERPRISE_VERSION options->addOptionChaining("security.redactClientLogData", @@ -563,10 +562,11 @@ Status storeServerOptions(const moe::Environment& params) { return {ErrorCodes::BadValue, "adaptiveThreadNum can only be used with serviceExecutor=adaptive"}; } - serverGlobalParams.adaptiveThreadNum = params["net.adaptiveThreadNum"].as(); - if (serverGlobalParams.adaptiveThreadNum < 1) { + const auto threadCount = params["net.adaptiveThreadNum"].as(); + if (threadCount < 1) { return {ErrorCodes::BadValue, "adaptiveThreadNum has to be at least 1"}; } + serverGlobalParams.adaptiveThreadNum = threadCount; } if (params.count("security.transitionToAuth")) { diff --git a/src/mongo/db/server_options_test.cpp b/src/mongo/db/server_options_test.cpp index 8eddeb88df..97a7dcc105 100644 --- a/src/mongo/db/server_options_test.cpp +++ b/src/mongo/db/server_options_test.cpp @@ -85,6 +85,46 @@ class OptionsParserTester : public moe::OptionsParser { std::string _config; }; +TEST(ServiceExecutorOptions, ThreadCountHasNoImplicitCommandLineValue) { + OptionsParserTester parser; + moe::Environment environment; + moe::OptionSection options; + ASSERT_OK(::mongo::addGeneralServerOptions(&options)); + ASSERT_OK(parser.run(options, {"binaryname"}, {}, &environment)); + // The runtime defaults to one adaptive worker, but an omitted option must not be + // mistaken for an explicit request to configure an adaptive-only setting. + ASSERT_FALSE(environment.count("net.adaptiveThreadNum")); + ASSERT_OK(::mongo::storeServerOptions(environment)); +} + +TEST(ServiceExecutorOptions, ExplicitThreadCountRequiresAdaptiveExecutor) { + OptionsParserTester parser; + moe::Environment environment; + moe::OptionSection options; + ASSERT_OK(::mongo::addGeneralServerOptions(&options)); + ASSERT_OK(parser.run(options, {"binaryname", "--adaptiveThreadNum=2"}, {}, &environment)); + const auto status = ::mongo::storeServerOptions(environment); + ASSERT_EQ(status.code(), ErrorCodes::BadValue); + ASSERT_STRING_CONTAINS(status.reason(), "serviceExecutor=adaptive"); +} + +TEST(ServiceExecutorOptions, RejectsNonpositiveAdaptiveThreadCounts) { + const auto oldExecutor = ::mongo::serverGlobalParams.serviceExecutor; + const auto restore = ::mongo::MakeGuard( + [&] { ::mongo::serverGlobalParams.serviceExecutor = oldExecutor; }); + for (const auto* count : {"--adaptiveThreadNum=0", "--adaptiveThreadNum=-1"}) { + OptionsParserTester parser; + moe::Environment environment; + moe::OptionSection options; + ASSERT_OK(::mongo::addGeneralServerOptions(&options)); + ASSERT_OK(parser.run( + options, {"binaryname", "--serviceExecutor=adaptive", count}, {}, &environment)); + const auto status = ::mongo::storeServerOptions(environment); + ASSERT_EQ(status.code(), ErrorCodes::BadValue); + ASSERT_STRING_CONTAINS(status.reason(), "at least 1"); + } +} + TEST(Verbosity, Default) { OptionsParserTester parser; moe::Environment environment; diff --git a/src/mongo/db/service_context.cpp b/src/mongo/db/service_context.cpp index aa18be58f3..de67138cda 100644 --- a/src/mongo/db/service_context.cpp +++ b/src/mongo/db/service_context.cpp @@ -243,15 +243,8 @@ ServiceContext::UniqueOperationContext ServiceContext::makeOperationContext(Clie auto opCtx = ObjectPool::newObjectRawPointer(client, _nextOpId.fetchAndAdd(1)); onCreate(opCtx, _clientObservers); - // if (!opCtx->lockState()) { - // MONGO_UNREACHABLE; - // opCtx->setLockState(std::make_unique()); - // } - // if (!opCtx->recoveryUnit()) { - // MONGO_UNREACHABLE; - // opCtx->setRecoveryUnit(new RecoveryUnitNoop(), - // WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); - // } + // Storage observers own locker/recovery-unit setup. In particular, Eloq's early + // catalog initialization intentionally leaves the recovery unit unset. { stdx::lock_guard lk(*client); client->setOperationContext(opCtx); diff --git a/src/mongo/db/service_context_d_test_fixture.cpp b/src/mongo/db/service_context_d_test_fixture.cpp index 1965c44a2a..99df564227 100644 --- a/src/mongo/db/service_context_d_test_fixture.cpp +++ b/src/mongo/db/service_context_d_test_fixture.cpp @@ -36,6 +36,7 @@ #include "mongo/db/catalog/catalog_control.h" #include "mongo/db/catalog/uuid_catalog.h" #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/logical_clock.h" #include "mongo/db/op_observer_registry.h" #include "mongo/db/service_entry_point_mongod.h" @@ -46,6 +47,30 @@ #include "mongo/db/catalog/database_holder.h" namespace mongo { +namespace { + +// Engine-backed MongoDB fixtures need real local locking. Install it only in these +// fixtures, after the production observer has initialized the operation context. +class MongoDTestLockerObserver final : public ServiceContext::ClientObserver { +public: + void onCreateClient(Client*) override {} + void onDestroyClient(Client*) override {} + void onCreateOperationContext(OperationContext* opCtx) override { + // Preserve Eloq's early catalog-initialization path before an engine exists. + if (!opCtx->getServiceContext()->getStorageEngine()) { + return; + } + auto locker = std::make_unique(); + if (opCtx->lockState()) { + opCtx->swapLockState(std::move(locker)); + } else { + opCtx->setLockState(std::move(locker)); + } + } + void onDestroyOperationContext(OperationContext*) override {} +}; + +} // namespace ServiceContextMongoDTest::ServiceContextMongoDTest() : ServiceContextMongoDTest("ephemeralForTest") {} @@ -54,7 +79,8 @@ ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine) : ServiceContextMongoDTest(engine, RepairAction::kNoRepair) {} ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine, RepairAction repair) - : _tempDir("service_context_d_test_fixture") { + : ServiceContextTest(std::make_unique()), + _tempDir("service_context_d_test_fixture") { _stashedStorageParams.engine = std::exchange(storageGlobalParams.engine, std::move(engine)); _stashedStorageParams.engineSetByUser = diff --git a/src/mongo/db/service_context_test_fixture.cpp b/src/mongo/db/service_context_test_fixture.cpp index e9aba48280..e86fec8fe1 100644 --- a/src/mongo/db/service_context_test_fixture.cpp +++ b/src/mongo/db/service_context_test_fixture.cpp @@ -38,9 +38,16 @@ namespace mongo { -ScopedGlobalServiceContextForTest::ScopedGlobalServiceContextForTest() { +ScopedGlobalServiceContextForTest::ScopedGlobalServiceContextForTest() + : ScopedGlobalServiceContextForTest(nullptr) {} + +ScopedGlobalServiceContextForTest::ScopedGlobalServiceContextForTest( + std::unique_ptr observer) { setGlobalServiceContext(ServiceContext::make()); auto const serviceContext = getGlobalServiceContext(); + if (observer) { + serviceContext->registerClientObserver(std::move(observer)); + } auto observerRegistry = std::make_unique(); serviceContext->setOpObserver(std::move(observerRegistry)); } @@ -53,7 +60,10 @@ ServiceContext* ScopedGlobalServiceContextForTest::getServiceContext() { return getGlobalServiceContext(); } -ServiceContextTest::ServiceContextTest() { +ServiceContextTest::ServiceContextTest() : ServiceContextTest(nullptr) {} + +ServiceContextTest::ServiceContextTest(std::unique_ptr observer) + : ScopedGlobalServiceContextForTest(std::move(observer)) { Client::initThread(getThreadName()); } diff --git a/src/mongo/db/service_context_test_fixture.h b/src/mongo/db/service_context_test_fixture.h index 8588b1e18e..b520dd34a2 100644 --- a/src/mongo/db/service_context_test_fixture.h +++ b/src/mongo/db/service_context_test_fixture.h @@ -45,6 +45,8 @@ class ScopedGlobalServiceContextForTest { protected: ScopedGlobalServiceContextForTest(); + explicit ScopedGlobalServiceContextForTest( + std::unique_ptr observer); virtual ~ScopedGlobalServiceContextForTest(); }; @@ -64,6 +66,8 @@ class ServiceContextTest : public unittest::Test, public ScopedGlobalServiceCont protected: ServiceContextTest(); + // Opt-in fixture observer, registered before the first client is created. + explicit ServiceContextTest(std::unique_ptr observer); virtual ~ServiceContextTest(); }; diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp index 4aa5eac97c..73e59c5a0c 100644 --- a/src/mongo/db/service_entry_point_common.cpp +++ b/src/mongo/db/service_entry_point_common.cpp @@ -982,6 +982,7 @@ void execCommandDatabase(OperationContext* opCtx, throw; } } catch (const DBException& e) { +#ifndef ELOQDOC_STANDALONE // If we got a stale config, wait in case the operation is stuck in a critical section if (auto sce = e.extraInfo()) { if (!opCtx->getClient()->isInDirectClient()) { @@ -1003,6 +1004,7 @@ void execCommandDatabase(OperationContext* opCtx, .ignore(); } } +#endif // Append the error labels for transient transaction errors. auto errorLabels = getErrorLabels(sessionOptions, command->getName(), e.code()); @@ -1159,6 +1161,7 @@ DbResponse receivedQuery(OperationContext* opCtx, dbResponse.exhaustNS = runQuery(opCtx, q, nss, dbResponse.response); } catch (const AssertionException& e) { +#ifndef ELOQDOC_STANDALONE // If we got a stale config, wait in case the operation is stuck in a critical section if (auto sce = e.extraInfo()) { if (!opCtx->getClient()->isInDirectClient()) { @@ -1168,6 +1171,7 @@ DbResponse receivedQuery(OperationContext* opCtx, .ignore(); } } +#endif dbResponse.response.reset(); generateLegacyQueryErrorResponse(e, q, &op, &dbResponse.response); diff --git a/src/mongo/db/standalone_runtime_test.cpp b/src/mongo/db/standalone_runtime_test.cpp new file mode 100644 index 0000000000..45a3835725 --- /dev/null +++ b/src/mongo/db/standalone_runtime_test.cpp @@ -0,0 +1,129 @@ +#include "mongo/platform/basic.h" + +#include "mongo/db/global_settings.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/repl/read_concern_args.h" +#include "mongo/db/repl/replication_coordinator_standalone.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { + +TEST(StandaloneRuntime, UsesGlobalReplicationSettings) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_TRUE(&coordinator.getSettings() == &getGlobalReplSettings()); +} + +TEST(StandaloneRuntime, DoesNotAdvertiseMongoReplication) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_FALSE(coordinator.isReplEnabled()); + ASSERT_EQUALS(repl::ReplicationCoordinator::modeNone, coordinator.getReplicationMode()); + ASSERT_TRUE(coordinator.isMasterForReportingPurposes()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, + coordinator.checkReplEnabledForCommand(nullptr).code()); +} + +TEST(StandaloneRuntime, AcceptsLocalReadsAndWrites) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + NamespaceString ns("test.collection"); + ASSERT_TRUE(coordinator.canAcceptWritesFor(nullptr, ns)); + ASSERT_OK(coordinator.checkCanServeReadsFor(nullptr, ns, false)); + ASSERT_TRUE(coordinator.buildsIndexes()); + ASSERT_FALSE(coordinator.shouldRelaxIndexConstraints(nullptr, ns)); +} + +TEST(StandaloneRuntime, PreservesExplicitDurabilityRequest) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + WriteConcernOptions concern; + concern.syncMode = WriteConcernOptions::SyncMode::JOURNAL; + ASSERT_TRUE(WriteConcernOptions::SyncMode::JOURNAL == + coordinator.populateUnsetWriteConcernOptionsSyncMode(concern).syncMode); + concern.syncMode = WriteConcernOptions::SyncMode::FSYNC; + ASSERT_TRUE(WriteConcernOptions::SyncMode::FSYNC == + coordinator.populateUnsetWriteConcernOptionsSyncMode(concern).syncMode); + concern.syncMode = WriteConcernOptions::SyncMode::UNSET; + ASSERT_TRUE(WriteConcernOptions::SyncMode::NONE == + coordinator.populateUnsetWriteConcernOptionsSyncMode(concern).syncMode); +} + +TEST(StandaloneRuntime, DoesNotWaitForNonexistentReplicas) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_OK(coordinator.awaitReplication(nullptr, repl::OpTime(), WriteConcernOptions()).status); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, + coordinator.checkIfWriteConcernCanBeSatisfied(WriteConcernOptions()).code()); +} + +TEST(StandaloneRuntime, AllowsNormalShutdownCommandToSkipStepDown) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_EQUALS(ErrorCodes::NotMaster, + coordinator.stepDown(nullptr, false, Seconds(0), Seconds(120)).code()); +} + +TEST(StandaloneRuntime, MaintenanceModeReturnsStatusWithoutThrowing) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + // ServiceEntryPointCommon's MaintenanceModeSetter deliberately ignores these errors. + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, coordinator.setMaintenanceMode(true).code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, coordinator.setMaintenanceMode(false).code()); + ASSERT_FALSE(coordinator.getMaintenanceMode()); +} + +TEST(StandaloneRuntime, TermOnNonReplicationCommandsReturnsBadValue) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + // Both find and getMore validate a supplied term through this Status-returning API. + ASSERT_EQUALS(ErrorCodes::BadValue, coordinator.updateTerm(nullptr, 1).code()); + ASSERT_EQUALS(ErrorCodes::BadValue, + coordinator.updateTerm(nullptr, repl::OpTime::kUninitializedTerm).code()); + ASSERT_TRUE(coordinator.getTerm() == repl::OpTime::kUninitializedTerm); +} + +TEST(StandaloneRuntime, IndexPrefetchServerParameterRoundTripsWithoutReplication) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + using Config = repl::ReplSettings::IndexPrefetchConfig; + ASSERT_TRUE(coordinator.getIndexPrefetchConfig() == Config::PREFETCH_ALL); + for (auto config : {Config::PREFETCH_NONE, Config::PREFETCH_ID_ONLY, Config::PREFETCH_ALL}) { + coordinator.setIndexPrefetchConfig(config); + ASSERT_TRUE(coordinator.getIndexPrefetchConfig() == config); + } + ASSERT_FALSE(coordinator.isReplEnabled()); +} + +TEST(StandaloneRuntime, ReplicationOnlyStatusAPIsReturnErrorsWithoutThrowing) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, + coordinator.waitForMemberState(repl::MemberState::RS_SECONDARY, + Milliseconds(0)).code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, + coordinator.setFollowerMode(repl::MemberState::RS_SECONDARY).code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, + coordinator.waitForDrainFinish(Milliseconds(0)).code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, coordinator.resyncData(nullptr, false).code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, + coordinator.prepareReplSetUpdatePositionCommand().getStatus().code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, coordinator.abortCatchupIfNeeded().code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, coordinator.stepUpIfEligible(false).code()); +} + +TEST(StandaloneRuntime, ReplicationOnlyVoidAPIsReportUnavailable) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_THROWS_CODE(coordinator.signalUpstreamUpdater(), + DBException, + ErrorCodes::NoReplicationEnabled); +} + +TEST(StandaloneRuntime, AcceptsUntimestampedLocalReadConcern) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_OK(coordinator.waitUntilOpTimeForRead(nullptr, repl::ReadConcernArgs())); + ASSERT_OK(coordinator.waitUntilOpTimeForRead( + nullptr, repl::ReadConcernArgs(repl::ReadConcernLevel::kAvailableReadConcern))); +} + +TEST(StandaloneRuntime, RejectsReplicaSetTimestampWaits) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + repl::ReadConcernArgs concern(repl::OpTime(Timestamp(1, 1), 1), + repl::ReadConcernLevel::kLocalReadConcern); + ASSERT_EQUALS(ErrorCodes::NotAReplicaSet, + coordinator.waitUntilOpTimeForRead(nullptr, concern).code()); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp index 3d212b0990..c8e5d5922f 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp @@ -37,6 +37,7 @@ #include "mongo/db/catalog/index_catalog_entry.h" #include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_recovery_unit.h" #include "mongo/db/storage/index_entry_comparison.h" +#include "mongo/db/storage/record_store.h" #include "mongo/stdx/memory.h" #include "mongo/util/mongoutils/str.h" @@ -138,7 +139,10 @@ class EphemeralForTestBtreeBuilderImpl : public SortedDataBuilderInterface { class EphemeralForTestBtreeImpl : public SortedDataInterface { public: - EphemeralForTestBtreeImpl(IndexSet* data, bool isUnique) : _data(data), _isUnique(isUnique) { + EphemeralForTestBtreeImpl(IndexSet* data, + bool isUnique, + std::unique_ptr recordStore) + : _data(data), _isUnique(isUnique), _recordStore(std::move(recordStore)) { _currentKeySize = 0; } @@ -223,9 +227,11 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { class Cursor final : public SortedDataInterface::Cursor { public: - Cursor(OperationContext* opCtx, const IndexSet& data, bool isForward, bool isUnique) + Cursor(OperationContext* opCtx, const IndexSet& data, bool isForward, bool isUnique, + RecordStore* recordStore) : _opCtx(opCtx), _data(data), + _recordStore(recordStore), _forward(isForward), _isUnique(isUnique), _it(data.end()) {} @@ -242,7 +248,7 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { if (_isEOF) return {}; - return *_it; + return currentEntry(); } void setEndPosition(const BSONObj& key, bool inclusive) override { @@ -278,7 +284,7 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { : compareKeys(_it->key, query) > 0); } - return *_it; + return currentEntry(); } boost::optional seek(const IndexSeekPoint& seekPoint, @@ -290,7 +296,7 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { if (_isEOF) return {}; dassert(compareKeys(_it->key, query) >= 0); - return *_it; + return currentEntry(); } void save() override { @@ -352,6 +358,18 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { } private: + IndexKeyEntry currentEntry() const { + auto entry = *_it; + if (_recordStore) { + // Eloq's IDHack path consumes the document returned with the index entry. + // Standalone index harnesses have no record store and still return keys only. + auto record = _recordStore->dataFor(_opCtx, entry.loc); + record.makeOwned(); + entry.record = std::move(record); + } + return entry; + } + bool atEndPoint() const { return _endState && _it == _endState->it; } @@ -446,6 +464,7 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { OperationContext* _opCtx; // not owned const IndexSet& _data; + RecordStore* const _recordStore; const bool _forward; const bool _isUnique; bool _isEOF = true; @@ -472,7 +491,13 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { virtual std::unique_ptr newCursor(OperationContext* opCtx, bool isForward) const { - return stdx::make_unique(opCtx, *_data, isForward, _isUnique); + return stdx::make_unique(opCtx, *_data, isForward, _isUnique, _recordStore.get()); + } + + SortedDataInterface::Cursor::UPtr newCursorPtr(OperationContext* opCtx, + bool isForward) const override { + return {newCursor(opCtx, isForward).release(), + [](SortedDataInterface::Cursor* cursor) { delete cursor; }}; } virtual Status initAsEmpty(OperationContext* opCtx) { @@ -503,6 +528,7 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { IndexSet* _data; long long _currentKeySize; const bool _isUnique; + const std::unique_ptr _recordStore; }; } // namespace @@ -511,11 +537,19 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { SortedDataInterface* getEphemeralForTestBtreeImpl(const Ordering& ordering, bool isUnique, std::shared_ptr* dataInOut) { + return getEphemeralForTestBtreeImpl(ordering, isUnique, dataInOut, nullptr); +} + +SortedDataInterface* getEphemeralForTestBtreeImpl(const Ordering& ordering, + bool isUnique, + std::shared_ptr* dataInOut, + std::unique_ptr recordStore) { invariant(dataInOut); if (!*dataInOut) { *dataInOut = std::make_shared(IndexEntryComparison(ordering)); } - return new EphemeralForTestBtreeImpl(static_cast(dataInOut->get()), isUnique); + return new EphemeralForTestBtreeImpl( + static_cast(dataInOut->get()), isUnique, std::move(recordStore)); } } // namespace mongo diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.h index 9d44e3e80c..e719097176 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.h +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.h @@ -36,6 +36,7 @@ namespace mongo { class IndexCatalogEntry; +class RecordStore; /** * Caller takes ownership. @@ -45,4 +46,9 @@ SortedDataInterface* getEphemeralForTestBtreeImpl(const Ordering& ordering, bool isUnique, std::shared_ptr* dataInOut); +SortedDataInterface* getEphemeralForTestBtreeImpl(const Ordering& ordering, + bool isUnique, + std::shared_ptr* dataInOut, + std::unique_ptr recordStore); + } // namespace mongo diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp index 0bc0f4f841..64b135b1c6 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp @@ -41,6 +41,73 @@ namespace mongo { +std::unique_ptr EphemeralForTestEngine::_getCatalogRecordStore() const { + stdx::lock_guard lk(_mutex); + auto it = _dataMap.find("_mdb_catalog"); + if (it == _dataMap.end() || !it->second) { + return nullptr; + } + auto data = it->second; + return stdx::make_unique("_mdb_catalog", &data); +} + +Status EphemeralForTestEngine::lockCollection(OperationContext* opCtx, + StringData ns, + bool isForWrite, + bool* exists, + std::string* version) { + auto catalog = _getCatalogRecordStore(); + RecordData record; + *exists = catalog && catalog->findRecord(opCtx, RecordId(ns.rawData(), ns.size()), &record); + version->clear(); + if (*exists) { + *version = record.toBson()["md"].Obj()["options"].Obj()["catalogVersion"].String(); + } + return Status::OK(); +} + +void EphemeralForTestEngine::listCollections(std::string_view dbName, + std::vector& out) const { + auto catalog = _getCatalogRecordStore(); + if (!catalog) { + return; + } + std::vector collections; + catalog->getAllCollections(collections); + for (const auto& ns : collections) { + if (NamespaceString(ns).db().toStringView() == dbName) { + out.push_back(ns); + } + } +} + +void EphemeralForTestEngine::listCollections(std::string_view dbName, + std::set& out) const { + std::vector collections; + listCollections(dbName, collections); + out.insert(collections.begin(), collections.end()); +} + +void EphemeralForTestEngine::listDatabases(std::vector& out) const { + auto catalog = _getCatalogRecordStore(); + if (!catalog) { + return; + } + std::vector collections; + catalog->getAllCollections(collections); + std::set databases; + for (const auto& ns : collections) { + databases.insert(NamespaceString(ns).db().toString()); + } + out.insert(out.end(), databases.begin(), databases.end()); +} + +bool EphemeralForTestEngine::databaseExists(std::string_view dbName) const { + std::vector collections; + listCollections(dbName, collections); + return !collections.empty(); +} + RecoveryUnit* EphemeralForTestEngine::newRecoveryUnit() { return new EphemeralForTestRecoveryUnit([this]() { stdx::lock_guard lk(_mutex); @@ -89,8 +156,11 @@ SortedDataInterface* EphemeralForTestEngine::getSortedDataInterface(OperationCon StringData ident, const IndexDescriptor* desc) { stdx::lock_guard lk(_mutex); + auto recordStore = stdx::make_unique( + desc->parentNS(), &_dataMap[desc->parentNS()]); return getEphemeralForTestBtreeImpl( - Ordering::make(desc->keyPattern()), desc->unique(), &_dataMap[ident]); + Ordering::make(desc->keyPattern()), desc->unique(), &_dataMap[ident], + std::move(recordStore)); } Status EphemeralForTestEngine::dropIdent(OperationContext* opCtx, StringData ident) { diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h index 38c9ea37ce..52406a1a9d 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h @@ -45,6 +45,23 @@ class EphemeralForTestEngine : public KVEngine { public: virtual RecoveryUnit* newRecoveryUnit(); + RecoveryUnit::UPtr newRecoveryUnitUPtr() override { + return {newRecoveryUnit(), [](RecoveryUnit* unit) { delete unit; }}; + } + + // Model the fork's namespace-keyed catalog in this in-memory test engine. Callers still + // use MongoDB's ordinary database/collection locks; there is no remote catalog cache. + Status lockCollection(OperationContext* opCtx, + StringData ns, + bool isForWrite, + bool* exists, + std::string* version) override; + void onAuthzDataChanged(OperationContext*) override {} + void listDatabases(std::vector& out) const override; + bool databaseExists(std::string_view dbName) const override; + void listCollections(std::string_view dbName, std::vector& out) const override; + void listCollections(std::string_view dbName, std::set& out) const override; + virtual Status createRecordStore(OperationContext* opCtx, StringData ns, StringData ident, @@ -115,6 +132,8 @@ class EphemeralForTestEngine : public KVEngine { } private: + std::unique_ptr _getCatalogRecordStore() const; + typedef StringMap> DataMap; mutable stdx::mutex _mutex; diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp index f2ed2f99de..9d985e0466 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp @@ -343,6 +343,19 @@ void EphemeralForTestRecordStore::deleteRecord(OperationContext* opCtx, const Re deleteRecord_inlock(opCtx, loc); } +void EphemeralForTestRecordStore::getAllCollections(std::vector& collections) const { + if (ns() != "_mdb_catalog") { + return; + } + stdx::lock_guard lock(_data->recordsMutex); + for (const auto& entry : _data->records) { + auto name = entry.second.toRecordData().toBson()["ns"]; + if (name.type() == String) { + collections.push_back(name.String()); + } + } +} + void EphemeralForTestRecordStore::deleteRecord_inlock(OperationContext* opCtx, const RecordId& loc) { EphemeralForTestRecord* rec = recordFor(loc); @@ -410,7 +423,16 @@ StatusWith EphemeralForTestRecordStore::insertRecord( memcpy(rec.data.get(), data, len); RecordId loc; - if (_data->isOplog) { + if (ns() == "_mdb_catalog") { + // Eloq's KVCatalog looks up metadata by namespace, not by a numeric record location. + BSONObj metadata(data); + StringData key = metadata["isFeatureDoc"].trueValue() + ? "featureDocument"_sd : metadata["ns"].checkAndGetStringData(); + loc = RecordId(key.rawData(), key.size()); + if (_data->records.count(loc)) { + return {ErrorCodes::DuplicateKey, "catalog namespace already exists"}; + } + } else if (_data->isOplog) { StatusWith status = extractAndCheckLocForOplog(data, len); if (!status.isOK()) return status; diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.h index 5e0d7ba1e5..e705027fe0 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.h +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.h @@ -64,6 +64,15 @@ class EphemeralForTestRecordStore : public RecordStore { virtual bool findRecord(OperationContext* opCtx, const RecordId& loc, RecordData* rd) const; + bool findRecord(OperationContext* opCtx, + const RecordId& loc, + RecordData* rd, + bool isForWrite) const override { + return findRecord(opCtx, loc, rd); + } + + void getAllCollections(std::vector& collections) const override; + virtual void deleteRecord(OperationContext* opCtx, const RecordId& dl); virtual StatusWith insertRecord( diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store_test.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store_test.cpp index 602311e4ce..fef4fe16be 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store_test.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store_test.cpp @@ -31,14 +31,140 @@ #include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.h" #include "mongo/base/init.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/index_names.h" +#include "mongo/db/operation_context_noop.h" +#include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h" #include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_recovery_unit.h" #include "mongo/db/storage/record_store_test_harness.h" +#include "mongo/db/storage/sorted_data_interface.h" #include "mongo/stdx/memory.h" #include "mongo/unittest/unittest.h" namespace mongo { namespace { +class NamespaceCatalogTest : public unittest::Test { +public: + NamespaceCatalogTest() : opCtx(engine.newRecoveryUnit()) { + ASSERT_OK(engine.createRecordStore(&opCtx, "_mdb_catalog", "_mdb_catalog", {})); + catalog = engine.getRecordStore(&opCtx, "_mdb_catalog", "_mdb_catalog", {}); + } + + StatusWith insert(const BSONObj& doc) { + return catalog->insertRecord(&opCtx, doc.objdata(), doc.objsize(), Timestamp(), false); + } + + BSONObj metadata() { + return BSON("ns" << "test.collection" << "md" + << BSON("options" << BSON("catalogVersion" << "version-1"))); + } + + EphemeralForTestEngine engine; + OperationContextNoop opCtx; + std::unique_ptr catalog; +}; + +TEST_F(NamespaceCatalogTest, NamespaceKeysAndEnumerationFollowTransactionRollback) { + { + WriteUnitOfWork wuow(&opCtx); + auto id = insert(metadata()); + ASSERT_OK(id.getStatus()); + ASSERT_EQ(RecordId("test.collection", 15), id.getValue()); + ASSERT_TRUE(engine.databaseExists("test")); + // Roll back this insertion. + } + ASSERT_FALSE(engine.databaseExists("test")); + { + WriteUnitOfWork wuow(&opCtx); + ASSERT_OK(insert(metadata()).getStatus()); + wuow.commit(); + } + std::vector databases; + engine.listDatabases(databases); + ASSERT_EQ(1U, databases.size()); + ASSERT_EQ("test", databases.front()); + std::vector collections; + engine.listCollections("test", collections); + ASSERT_EQ(1U, collections.size()); + ASSERT_EQ("test.collection", collections.front()); + std::set names; + engine.listCollections("test", names); + ASSERT_EQ(1U, names.count("test.collection")); + ASSERT_FALSE(engine.databaseExists("other")); + + bool exists = false; + std::string version; + ASSERT_OK(engine.lockCollection(&opCtx, "test.collection", false, &exists, &version)); + ASSERT_TRUE(exists); + ASSERT_EQ("version-1", version); + ASSERT_OK(engine.lockCollection(&opCtx, "test.missing", true, &exists, &version)); + ASSERT_FALSE(exists); + ASSERT_TRUE(version.empty()); +} + +TEST_F(NamespaceCatalogTest, DuplicateInsertionAndAbortedDeletionPreserveMetadata) { + { + WriteUnitOfWork wuow(&opCtx); + ASSERT_OK(insert(metadata()).getStatus()); + wuow.commit(); + } + { + WriteUnitOfWork wuow(&opCtx); + ASSERT_EQ(ErrorCodes::DuplicateKey, insert(metadata()).getStatus().code()); + } + { + WriteUnitOfWork wuow(&opCtx); + catalog->deleteRecord(&opCtx, RecordId("test.collection", 15)); + ASSERT_FALSE(engine.databaseExists("test")); + } + ASSERT_TRUE(engine.databaseExists("test")); + { + WriteUnitOfWork wuow(&opCtx); + catalog->deleteRecord(&opCtx, RecordId("test.collection", 15)); + wuow.commit(); + } + ASSERT_FALSE(engine.databaseExists("test")); +} + +TEST_F(NamespaceCatalogTest, FeatureDocumentDoesNotAppearAsACollection) { + WriteUnitOfWork wuow(&opCtx); + auto id = insert(BSON("isFeatureDoc" << true << "ns" << BSONNULL)); + ASSERT_OK(id.getStatus()); + ASSERT_EQ(RecordId("featureDocument", 15), id.getValue()); + RecordData data; + ASSERT_TRUE(catalog->findRecord(&opCtx, id.getValue(), &data, true)); + std::vector collections; + catalog->getAllCollections(collections); + ASSERT_TRUE(collections.empty()); + wuow.commit(); +} + +TEST_F(NamespaceCatalogTest, PooledIndexCursorReturnsDocumentForIDHack) { + auto records = engine.getRecordStore(&opCtx, "test.collection", "test.collection", {}); + IndexDescriptor descriptor(nullptr, IndexNames::BTREE, + BSON("v" << 2 << "key" << BSON("_id" << 1) << "name" << "_id_" + << "ns" << "test.collection" << "unique" << true)); + ASSERT_OK(engine.createSortedDataInterface(&opCtx, "test.collection._id_", &descriptor)); + std::unique_ptr index( + engine.getSortedDataInterface(&opCtx, "test.collection._id_", &descriptor)); + BSONObj document = BSON("_id" << 42 << "payload" << "record from index lookup"); + { + WriteUnitOfWork wuow(&opCtx); + auto id = records->insertRecord(&opCtx, document.objdata(), document.objsize(), + Timestamp(), false); + ASSERT_OK(id.getStatus()); + ASSERT_OK(index->insert(&opCtx, BSON("" << 42), id.getValue(), false)); + wuow.commit(); + } + auto cursor = index->newCursorPtr(&opCtx); + auto entry = cursor->seekExact(BSON("" << 42)); + ASSERT_TRUE(entry); + ASSERT_TRUE(entry->record); + ASSERT_EQ(42, entry->record->toBson()["_id"].Int()); + ASSERT_EQ("record from index lookup", entry->record->toBson()["payload"].String()); +} + class EphemeralForTestHarnessHelper final : public RecordStoreHarnessHelper { public: EphemeralForTestHarnessHelper() {} diff --git a/src/mongo/db/storage/key_string_test.cpp b/src/mongo/db/storage/key_string_test.cpp index 0d87d0dfea..b0226e8121 100644 --- a/src/mongo/db/storage/key_string_test.cpp +++ b/src/mongo/db/storage/key_string_test.cpp @@ -103,8 +103,9 @@ TEST_F(KeyStringTest, Simple1) { ASSERT_BSONOBJ_LT(a, b); - ASSERT_LESS_THAN(KeyString(version, a, ALL_ASCENDING, RecordId()), - KeyString(version, b, ALL_ASCENDING, RecordId())); + // These keys use the integer RecordId format, including its zero sentinel. + ASSERT_LESS_THAN(KeyString(version, a, ALL_ASCENDING, RecordId(0)), + KeyString(version, b, ALL_ASCENDING, RecordId(0))); } #define ROUNDTRIP_ORDER(version, x, order) \ @@ -445,7 +446,7 @@ TEST_F(KeyStringTest, RecordIdOrder1) { KeyString a(version, BSON("" << 5), ordering, RecordId::min()); KeyString b(version, BSON("" << 5), ordering, RecordId(2)); KeyString c(version, BSON("" << 5), ordering, RecordId(3)); - KeyString d(version, BSON("" << 6), ordering, RecordId()); + KeyString d(version, BSON("" << 6), ordering, RecordId(0)); KeyString e(version, BSON("" << 6), ordering, RecordId(1)); ASSERT_LESS_THAN(a, b); @@ -1124,7 +1125,7 @@ TEST_F(KeyStringTest, RecordIds) { } if (rid.isNormal()) { - ASSERT_GT(ks, KeyString(version, RecordId())); + ASSERT_GT(ks, KeyString(version, RecordId(0))); ASSERT_GT(ks, KeyString(version, RecordId::min())); ASSERT_LT(ks, KeyString(version, RecordId::max())); diff --git a/src/mongo/db/storage/kv/SConscript b/src/mongo/db/storage/kv/SConscript index ea5b289b1e..786a502eef 100644 --- a/src/mongo/db/storage/kv/SConscript +++ b/src/mongo/db/storage/kv/SConscript @@ -134,8 +134,7 @@ env.CppUnitTest( '$BUILD_DIR/mongo/db/catalog/collection_options', '$BUILD_DIR/mongo/db/namespace_string', '$BUILD_DIR/mongo/db/service_context_test_fixture', - '$BUILD_DIR/mongo/db/storage/devnull/storage_devnull_core', - '$BUILD_DIR/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store', + '$BUILD_DIR/mongo/db/storage/ephemeral_for_test/storage_ephemeral_for_test_core', ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/auth/authmocks' diff --git a/src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp b/src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp index b27937839b..cd429f9610 100644 --- a/src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp +++ b/src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp @@ -38,7 +38,7 @@ #include "mongo/db/index_names.h" #include "mongo/db/operation_context_noop.h" #include "mongo/db/service_context_test_fixture.h" -#include "mongo/db/storage/devnull/devnull_kv_engine.h" +#include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h" #include "mongo/db/storage/kv/kv_database_catalog_entry_mock.h" #include "mongo/db/storage/kv/kv_engine.h" #include "mongo/db/storage/kv/kv_storage_engine.h" @@ -51,31 +51,35 @@ namespace { class KVCollectionCatalogEntryTest : public ServiceContextTest { public: - KVCollectionCatalogEntryTest() - : _nss("unittests.kv_collection_catalog_entry"), - _storageEngine( - new DevNullKVEngine(), KVStorageEngineOptions(), kvDatabaseCatalogEntryMockFactory) { - _storageEngine.finishInit(); + KVCollectionCatalogEntryTest() : _nss("unittests.kv_collection_catalog_entry") { + auto engine = std::make_unique( + new EphemeralForTestEngine(), KVStorageEngineOptions(), kvDatabaseCatalogEntryMockFactory); + _storageEngine = engine.get(); + getServiceContext()->setStorageEngine(std::move(engine)); + _storageEngine->finishInit(); } ~KVCollectionCatalogEntryTest() { - _storageEngine.cleanShutdown(); + _storageEngine->cleanShutdown(); } std::unique_ptr newOperationContext() { - return stdx::make_unique(_storageEngine.newRecoveryUnit()); + auto opCtx = stdx::make_unique(getClient(), 0); + opCtx->setRecoveryUnit(_storageEngine->newRecoveryUnit(), + WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); + return opCtx; } void setUp() final { auto opCtx = newOperationContext(); DatabaseCatalogEntry* dbEntry = - _storageEngine.getDatabaseCatalogEntry(opCtx.get(), _nss.db()); + _storageEngine->getDatabaseCatalogEntry(opCtx.get(), _nss.db()); { WriteUnitOfWork wuow(opCtx.get()); - const bool allocateDefaultSpace = true; - ASSERT_OK(dbEntry->createCollection( - opCtx.get(), _nss.ns(), CollectionOptions(), allocateDefaultSpace)); + auto idIndex = BSON("v" << 2 << "key" << BSON("_id" << 1) << "name" << "_id_" + << "ns" << _nss.ns()); + ASSERT_OK(dbEntry->createCollection(opCtx.get(), _nss, CollectionOptions(), idIndex)); wuow.commit(); } } @@ -83,8 +87,8 @@ class KVCollectionCatalogEntryTest : public ServiceContextTest { CollectionCatalogEntry* getCollectionCatalogEntry() { auto opCtx = newOperationContext(); DatabaseCatalogEntry* dbEntry = - _storageEngine.getDatabaseCatalogEntry(opCtx.get(), _nss.db()); - return dbEntry->getCollectionCatalogEntry(_nss.ns()); + _storageEngine->getDatabaseCatalogEntry(opCtx.get(), _nss.db()); + return dbEntry->getCollectionCatalogEntry(opCtx.get(), _nss.ns()); } std::string createIndex(BSONObj keyPattern, std::string indexType = IndexNames::BTREE) { @@ -136,7 +140,7 @@ class KVCollectionCatalogEntryTest : public ServiceContextTest { } const NamespaceString _nss; - KVStorageEngine _storageEngine; + KVStorageEngine* _storageEngine; size_t numIndexesCreated = 0; }; diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp index d9623f23d9..bb2bdd3a75 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp @@ -279,11 +279,11 @@ void KVStorageEngine::closeCatalog(OperationContext* opCtx) { _dumpCatalog(opCtx); } - // stdx::lock_guard lock(_dbsLock); - // for (auto entry : _dbMapVector) { - // delete entry.second; - // } - _dbMapVector.clear(); + // The slots are indexed by LocalThread::ID() when the catalog is reopened. + // Clear their entries, not the slots themselves. + for (auto& dbMap : _dbMapVector) { + dbMap.clear(); + } _catalog.reset(nullptr); _catalogRecordStore.reset(nullptr); diff --git a/src/mongo/db/storage/storage_engine_init.cpp b/src/mongo/db/storage/storage_engine_init.cpp index fc31b4e44c..92e4e267ad 100644 --- a/src/mongo/db/storage/storage_engine_init.cpp +++ b/src/mongo/db/storage/storage_engine_init.cpp @@ -36,7 +36,6 @@ #include "mongo/base/init.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/concurrency/eloq_locker_noop.h" -#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/operation_context.h" #include "mongo/db/storage/storage_engine_lock_file.h" #include "mongo/db/storage/storage_engine_metadata.h" @@ -336,8 +335,7 @@ class StorageClientObserver final : public ServiceContext::ClientObserver { void onDestroyClient(Client* client) override{}; void onCreateOperationContext(OperationContext* opCtx) override { auto service = opCtx->getServiceContext(); - // Eloq TableSchema require a OperationContext while storageEngine is not set. - // Acutally, we always use EloqLockerNoop. + // Eloq TableSchema needs an OperationContext even before its engine is installed. auto storageEngine = service->getStorageEngine(); // NOTE(schwerin): The following uassert would be more desirable than the early return when // no storage engine is set, but to achieve that we would have to ensure that this file was @@ -348,10 +346,17 @@ class StorageClientObserver final : public ServiceContext::ClientObserver { // storageEngine); - if (opCtx->lockState()) { + if (dynamic_cast(opCtx->lockState())) { opCtx->resetLockState(); } else { - opCtx->setLockState(stdx::make_unique()); + // Production always uses Eloq locking. A pooled context may carry a test + // fixture's locker; replace it rather than keeping it in a server context. + auto locker = stdx::make_unique(); + if (opCtx->lockState()) { + opCtx->swapLockState(std::move(locker)); + } else { + opCtx->setLockState(std::move(locker)); + } } if (storageEngine) { diff --git a/src/mongo/db/storage/test_harness_helper.h b/src/mongo/db/storage/test_harness_helper.h index cd21223be1..2fcb909ec9 100644 --- a/src/mongo/db/storage/test_harness_helper.h +++ b/src/mongo/db/storage/test_harness_helper.h @@ -32,6 +32,7 @@ #include #include +#include "mongo/db/concurrency/locker_noop.h" #include "mongo/db/jsobj.h" #include "mongo/db/operation_context_noop.h" #include "mongo/db/record_id.h" @@ -50,6 +51,11 @@ class HarnessHelper { virtual ServiceContext::UniqueOperationContext newOperationContext(Client* const client) { auto opCtx = client->makeOperationContext(); + // Engine harnesses supply their own recovery units, without installing a server + // storage observer. WriteUnitOfWork still needs a locker on fresh/pooled contexts. + if (!opCtx->lockState()) { + opCtx->setLockState(stdx::make_unique()); + } opCtx->setRecoveryUnit(newRecoveryUnit().release(), WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); return opCtx; diff --git a/src/mongo/db/update/update_driver_test.cpp b/src/mongo/db/update/update_driver_test.cpp index cfa3241ce1..cbbdcaf48c 100644 --- a/src/mongo/db/update/update_driver_test.cpp +++ b/src/mongo/db/update/update_driver_test.cpp @@ -169,10 +169,10 @@ class CreateFromQueryFixture : public mongo::unittest::Test { : _opCtx(_serviceContext.makeOperationContext()), // _driverOps(new UpdateDriver(new ExpressionContext(_opCtx.get(), nullptr))), _driverOps(new UpdateDriver( - ObjectPool::newObjectRawPointer(_opCtx, nullptr))), + ObjectPool::newObjectRawPointer(_opCtx.get(), nullptr))), // _driverRepl(new UpdateDriver(new ExpressionContext(_opCtx.get(), nullptr))) _driverRepl(new UpdateDriver( - ObjectPool::newObjectRawPointer(_opCtx, nullptr))) { + ObjectPool::newObjectRawPointer(_opCtx.get(), nullptr))) { _driverOps->parse(fromjson("{$set:{'_':1}}"), _arrayFilters).transitional_ignore(); _driverRepl->parse(fromjson("{}"), _arrayFilters).transitional_ignore(); } diff --git a/src/mongo/platform/stack_locator.cpp b/src/mongo/platform/stack_locator.cpp index 3668615611..e937ba5783 100644 --- a/src/mongo/platform/stack_locator.cpp +++ b/src/mongo/platform/stack_locator.cpp @@ -30,30 +30,25 @@ #include "mongo/platform/stack_locator.h" +#include #include "mongo/util/assert_util.h" namespace mongo { boost::optional StackLocator::available() const { - // Returns a disengaged optional since the remaining stack cannot be determined. - return boost::none; - if (!begin() || !end()) return boost::none; - // Technically, it is undefined behavior to compare or subtract - // two pointers that do not point into the same - // aggregate. However, we know that these are both pointers within - // the same stack, and it seems unlikely that the compiler will - // see that it can elide the comparison here. - - const auto cbegin = reinterpret_cast(begin()); - const auto cthis = reinterpret_cast(this); - const auto cend = reinterpret_cast(end()); + // Compare addresses as integers: coroutine and native stack pointers need not + // point into the same allocation. + const auto cbegin = reinterpret_cast(begin()); + const auto cthis = reinterpret_cast(this); + const auto cend = reinterpret_cast(end()); - // TODO: Assumes that stack grows downward - invariant(cthis <= cbegin); - invariant(cthis > cend); + // A coroutine may be running on a different stack from the OS thread whose + // bounds we queried. Do not report (or assert on) those unrelated bounds. + if (cthis > cbegin || cthis <= cend) + return boost::none; std::size_t avail = cthis - cend; diff --git a/src/mongo/s/shard_id_test.cpp b/src/mongo/s/shard_id_test.cpp index 6393c9e9a9..2914ce6e10 100644 --- a/src/mongo/s/shard_id_test.cpp +++ b/src/mongo/s/shard_id_test.cpp @@ -86,8 +86,11 @@ TEST(ShardId, Compare) { ShardId sa1(a1); ShardId sb(b); ASSERT_EQUALS(sa.compare(sa1), a.compare(a1)); - ASSERT_EQUALS(sb.compare(sa1), b.compare(a1)); - ASSERT_EQUALS(sa.compare(sb), a.compare(b)); + // String comparisons promise the sign, not a particular negative/positive magnitude. + ASSERT_EQUALS(sb.compare(sa1) < 0, b.compare(a1) < 0); + ASSERT_EQUALS(sb.compare(sa1) > 0, b.compare(a1) > 0); + ASSERT_EQUALS(sa.compare(sb) < 0, a.compare(b) < 0); + ASSERT_EQUALS(sa.compare(sb) > 0, a.compare(b) > 0); } TEST(ShardId, Equals) { diff --git a/src/mongo/transport/service_executor_adaptive.cpp b/src/mongo/transport/service_executor_adaptive.cpp index 5906a1b3cb..7b65f077c3 100644 --- a/src/mongo/transport/service_executor_adaptive.cpp +++ b/src/mongo/transport/service_executor_adaptive.cpp @@ -188,6 +188,9 @@ ServiceExecutorAdaptive::~ServiceExecutorAdaptive() { Status ServiceExecutorAdaptive::start() { invariant(!_isRunning.load()); + if (_reactorHandles.empty()) { + return {ErrorCodes::InvalidOptions, "Adaptive executor requires at least one reactor"}; + } _isRunning.store(true); // _controllerThread = stdx::thread(&ServiceExecutorAdaptive::_controllerThreadRoutine, this); for (auto i = 0; i < _reactorHandles.size(); i++) { @@ -204,7 +207,9 @@ Status ServiceExecutorAdaptive::shutdown(Milliseconds timeout) { _isRunning.store(false); _scheduleCondition.notify_one(); - _controllerThread.join(); + if (_controllerThread.joinable()) { + _controllerThread.join(); + } stdx::unique_lock lk(_threadsMutex); for (auto& reactorHandle : _reactorHandles) { @@ -222,78 +227,9 @@ Status ServiceExecutorAdaptive::shutdown(Milliseconds timeout) { Status ServiceExecutorAdaptive::schedule(ServiceExecutorAdaptive::Task task, ScheduleFlags flags, ServiceExecutorTaskName taskName) { + // EloqDoc uses this executor only to run ingress reactors. Session tasks are + // assigned to ServiceExecutorCoroutine by ServiceEntryPointImpl::startSession(). MONGO_UNREACHABLE; - auto scheduleTime = _tickSource->getTicks(); - auto pendingCounterPtr = (flags & kDeferredTask) ? &_deferredTasksQueued : &_tasksQueued; - pendingCounterPtr->addAndFetch(1); - - if (!_isRunning.load()) { - return {ErrorCodes::ShutdownInProgress, "Executor is not running"}; - } - - auto wrappedTask = - [this, task = std::move(task), scheduleTime, pendingCounterPtr, taskName, flags] { - pendingCounterPtr->subtractAndFetch(1); - auto start = _tickSource->getTicks(); - _totalSpentQueued.addAndFetch(start - scheduleTime); - - _localThreadState->threadMetrics[static_cast(taskName)] - ._totalSpentQueued.addAndFetch(start - scheduleTime); - - if (_localThreadState->recursionDepth++ == 0) { - _localThreadState->executing.markRunning(); - _threadsInUse.addAndFetch(1); - } - const auto guard = MakeGuard([this, taskName] { - if (--_localThreadState->recursionDepth == 0) { - _localThreadState->executingCurRun += - _localThreadState->executing.markStopped(); - _threadsInUse.subtractAndFetch(1); - } - _totalExecuted.addAndFetch(1); - _localThreadState->threadMetrics[static_cast(taskName)] - ._totalExecuted.addAndFetch(1); - }); - - TickTimer _localTimer(_tickSource); - task(); - _localThreadState->threadMetrics[static_cast(taskName)] - ._totalSpentExecuting.addAndFetch(_localTimer.sinceStartTicks()); - - if ((flags & ServiceExecutor::kMayYieldBeforeSchedule) && - (_localThreadState->markIdleCounter++ & 0xf)) { - markThreadIdle(); - } - }; - - // Dispatching a task on the io_context will run the task immediately, and may run it - // on the current thread (if the current thread is running the io_context right now). - // - // Posting a task on the io_context will run the task without recursion. - // - // If the task is allowed to recurse and we are not over the depth limit, dispatch it so it - // can be called immediately and recursively. - if ((flags & kMayRecurse) && - (_localThreadState->recursionDepth + 1 < _config->recursionLimit())) { - // _reactorHandle->schedule(Reactor::kDispatch, std::move(wrappedTask)); - } else { - // _reactorHandle->schedule(Reactor::kPost, std::move(wrappedTask)); - } - - _lastScheduleTimer.reset(); - _totalQueued.addAndFetch(1); - - _accumulatedMetrics[static_cast(taskName)]._totalQueued.addAndFetch(1); - - // Deferred tasks never count against the thread starvation avoidance. For other tasks, we - // notify the controller thread that a task has been scheduled and we should monitor thread - // starvation. - if (_isStarved() && !(flags & kDeferredTask)) { - _starvationCheckRequests.addAndFetch(1); - _scheduleCondition.notify_one(); - } - - return Status::OK(); } bool ServiceExecutorAdaptive::_isStarved() const { diff --git a/src/mongo/transport/service_executor_adaptive.h b/src/mongo/transport/service_executor_adaptive.h index 35ba172e2b..ea13114d37 100644 --- a/src/mongo/transport/service_executor_adaptive.h +++ b/src/mongo/transport/service_executor_adaptive.h @@ -185,7 +185,7 @@ class ServiceExecutorAdaptive : public ServiceExecutor { ThreadState(TickSource* ts) : running(ts), executing(ts) {} CumulativeTickTimer running; - TickSource::Tick executingCurRun; + TickSource::Tick executingCurRun = 0; CumulativeTickTimer executing; MetricsArray threadMetrics; std::int64_t markIdleCounter = 0; diff --git a/src/mongo/transport/service_executor_test.cpp b/src/mongo/transport/service_executor_test.cpp index da2f6a867d..49a67e75c2 100644 --- a/src/mongo/transport/service_executor_test.cpp +++ b/src/mongo/transport/service_executor_test.cpp @@ -32,16 +32,27 @@ #include "boost/optional.hpp" +#include "mongo/db/local_thread_state.h" #include "mongo/db/service_context.h" +#include "mongo/stdx/future.h" #include "mongo/transport/service_executor_adaptive.h" +#include "mongo/transport/service_executor_coroutine.h" #include "mongo/transport/service_executor_synchronous.h" #include "mongo/transport/service_executor_task_names.h" +#include "mongo/unittest/death_test.h" #include "mongo/unittest/unittest.h" #include "mongo/util/log.h" #include "mongo/util/scopeguard.h" #include +#ifdef ELOQ_MODULE_ENABLED +#include +#include + +DECLARE_bool(brpc_worker_as_ext_processor); +#endif + namespace mongo { namespace { using namespace transport; @@ -92,7 +103,8 @@ class ASIOReactor : public transport::Reactor { ASIOReactor() : _ioContext() {} void run() noexcept final { - MONGO_UNREACHABLE; + asio::io_context::work work(_ioContext); + _ioContext.run(); } void runFor(Milliseconds time) noexcept final { @@ -113,7 +125,7 @@ class ASIOReactor : public transport::Reactor { void drain() override final { _ioContext.restart(); while (_ioContext.poll()) { - LOG(1) << "Draining remaining work in reactor."; + MONGO_LOG(1) << "Draining remaining work in reactor."; } _ioContext.stop(); } @@ -152,15 +164,20 @@ class ServiceExecutorAdaptiveFixture : public unittest::Test { auto scOwned = ServiceContext::make(); setGlobalServiceContext(std::move(scOwned)); - auto configOwned = stdx::make_unique(); - executorConfig = configOwned.get(); + reactors = {std::make_shared(), std::make_shared()}; executor = stdx::make_unique( - getGlobalServiceContext(), std::make_shared(), std::move(configOwned)); + getGlobalServiceContext(), std::vector(reactors), + stdx::make_unique()); + } + + void tearDown() override { + ASSERT_OK(executor->shutdown(kShutdownTime)); + executor.reset(); + setGlobalServiceContext({}); } - ServiceExecutorAdaptive::Options* executorConfig; + std::vector reactors; std::unique_ptr executor; - std::shared_ptr asioIOCtx; }; class ServiceExecutorSynchronousFixture : public unittest::Test { @@ -176,34 +193,130 @@ class ServiceExecutorSynchronousFixture : public unittest::Test { }; void scheduleBasicTask(ServiceExecutor* exec, bool expectSuccess) { - stdx::condition_variable cond; - stdx::mutex mutex; - auto task = [&cond, &mutex] { - stdx::unique_lock lk(mutex); - cond.notify_all(); + struct Completion { + stdx::condition_variable cond; + stdx::mutex mutex; + bool completed = false; + }; + auto completion = std::make_shared(); + auto task = [completion] { + stdx::unique_lock lk(completion->mutex); + completion->completed = true; + completion->cond.notify_all(); }; - stdx::unique_lock lk(mutex); + stdx::unique_lock lk(completion->mutex); auto status = exec->schedule( std::move(task), ServiceExecutor::kEmptyFlags, ServiceExecutorTaskName::kSSMStartSession); if (expectSuccess) { ASSERT_OK(status); - cond.wait(lk); + ASSERT_TRUE(completion->cond.wait_for( + lk, std::chrono::seconds(10), [&] { return completion->completed; })); } else { ASSERT_NOT_OK(status); } } -TEST_F(ServiceExecutorAdaptiveFixture, BasicTaskRuns) { +TEST_F(ServiceExecutorAdaptiveFixture, RunsEachIngressReactor) { ASSERT_OK(executor->start()); - auto guard = MakeGuard([this] { ASSERT_OK(executor->shutdown(kShutdownTime)); }); + for (const auto& reactor : reactors) { + auto completed = std::make_shared>(); + auto future = completed->get_future(); + reactor->schedule(Reactor::kPost, [completed] { completed->set_value(); }); + ASSERT_TRUE(future.wait_for(std::chrono::seconds(10)) == stdx::future_status::ready); + future.get(); + } + ASSERT_EQ(reactors.size(), static_cast(executor->threadsRunning())); + ASSERT_OK(executor->shutdown(kShutdownTime)); + ASSERT_EQ(0, executor->threadsRunning()); +} - scheduleBasicTask(executor.get(), true); +TEST_F(ServiceExecutorAdaptiveFixture, ShutdownBeforeStartupIsSafe) { + ASSERT_OK(executor->shutdown(kShutdownTime)); } -TEST_F(ServiceExecutorAdaptiveFixture, ScheduleFailsBeforeStartup) { - scheduleBasicTask(executor.get(), false); +TEST_F(ServiceExecutorAdaptiveFixture, ShutdownIsIdempotent) { + ASSERT_OK(executor->start()); + ASSERT_OK(executor->shutdown(kShutdownTime)); + ASSERT_OK(executor->shutdown(kShutdownTime)); +} + +TEST_F(ServiceExecutorAdaptiveFixture, RejectsEmptyReactorList) { + ServiceExecutorAdaptive empty(getGlobalServiceContext(), {}, + stdx::make_unique()); + ASSERT_EQ(ErrorCodes::InvalidOptions, empty.start().code()); + ASSERT_OK(empty.shutdown(kShutdownTime)); +} + +DEATH_TEST_F(ServiceExecutorAdaptiveFixture, SchedulingIsUnreachable, "MONGO_UNREACHABLE") { + ASSERT_OK(executor->start()); + executor->schedule([] {}, ServiceExecutor::kEmptyFlags, + ServiceExecutorTaskName::kSSMStartSession).ignore(); +} + +#ifdef ELOQ_MODULE_ENABLED +TEST(ServiceExecutorCoroutine, ModuleSchedulingAndLifecycle) { + constexpr int kThreadGroups = 2; + // Start only brpc's worker pool: no Data Substrate, storage engine, or data-store fixture. + // This matches the module-backed scheduler used by the EloqDoc server. + FLAGS_brpc_worker_as_ext_processor = true; + ASSERT_EQ(0, bthread_setconcurrency(kThreadGroups)); + bthread_t bootstrap; + ASSERT_EQ(0, bthread_start_background( + &bootstrap, nullptr, [](void*) -> void* { return nullptr; }, nullptr)); + ASSERT_EQ(0, bthread_join(bootstrap, nullptr)); + + auto service = ServiceContext::make(); + ServiceExecutorCoroutine executor(service.get(), kThreadGroups); + bool rejectedTaskRan = false; + auto rejectedTask = [&] { rejectedTaskRan = true; }; + const auto flags = ServiceExecutor::kEmptyFlags; + const auto taskName = ServiceExecutorTaskName::kSSMStartSession; + ASSERT_EQ(ErrorCodes::ShutdownInProgress, + executor.schedule(rejectedTask, flags, taskName).code()); + ASSERT_EQ(ErrorCodes::ShutdownInProgress, + executor.schedule(rejectedTask, flags, taskName, 1).code()); + + ASSERT_OK(executor.start()); + bool running = true; + auto stop = MakeGuard([&] { + if (running) + ASSERT_OK(executor.shutdown(kShutdownTime)); + }); + auto checkGroup = [](int16_t expectedGroup, auto submit) { + auto completed = std::make_shared>(); + auto future = completed->get_future(); + ServiceExecutor::Task task = [completed] { completed->set_value(LocalThread::ID()); }; + submit(task); + ASSERT_TRUE(future.wait_for(std::chrono::seconds(10)) == stdx::future_status::ready); + ASSERT_EQ(expectedGroup, future.get()); + }; + checkGroup(0, [&](const ServiceExecutor::Task& task) { + ASSERT_OK(executor.schedule(task, flags, taskName)); + }); + for (int16_t group = 0; group < kThreadGroups; ++group) { + checkGroup(group, [&](const ServiceExecutor::Task& task) { + ASSERT_OK(executor.schedule(task, flags, taskName, group)); + }); + checkGroup(group, [&](const ServiceExecutor::Task& task) { + executor.coroutineResumeFunctor(group, task)(); + }); + checkGroup(group, [&](const ServiceExecutor::Task& task) { + executor.coroutineLongResumeFunctor(group, task)(); + }); + checkGroup(group, [&](const ServiceExecutor::Task& task) { + executor.deferCallOnMainStack(group, ServiceExecutor::Task(task)); + }); + } + ASSERT_OK(executor.shutdown(kShutdownTime)); + running = false; + ASSERT_EQ(ErrorCodes::ShutdownInProgress, + executor.schedule(rejectedTask, flags, taskName).code()); + ASSERT_EQ(ErrorCodes::ShutdownInProgress, + executor.schedule(rejectedTask, flags, taskName, 1).code()); + ASSERT_FALSE(rejectedTaskRan); } +#endif TEST_F(ServiceExecutorSynchronousFixture, BasicTaskRuns) { ASSERT_OK(executor->start()); diff --git a/src/mongo/transport/service_state_machine.cpp b/src/mongo/transport/service_state_machine.cpp index e60ffa4185..d539bf6126 100644 --- a/src/mongo/transport/service_state_machine.cpp +++ b/src/mongo/transport/service_state_machine.cpp @@ -58,6 +58,7 @@ #include #include #include +#include namespace mongo { namespace { @@ -315,6 +316,7 @@ void ServiceStateMachine::reset(ServiceContext* svcContext, _coroLongResume = {}; _coroMigrateThreadGroup = {}; _resumeTask = {}; + _dbResponse = {}; _migrating.store(false, std::memory_order_relaxed); _threadGroupId.store(groupId, std::memory_order_relaxed); _owned.store(Ownership::kUnowned); @@ -467,7 +469,6 @@ void ServiceStateMachine::_processMessage(ThreadGuard guard) { // Pass sourced Message to handler to generate response. - DbResponse dbresponse; { // OperationContext opCtx(Client::getCurrent(), _serviceContext->nextOpId()); // _serviceContext->initOperationContext(&opCtx); @@ -477,11 +478,12 @@ void ServiceStateMachine::_processMessage(ThreadGuard guard) { // The handleRequest is implemented in a subclass for mongod/mongos and actually all the // database work for this request. - dbresponse = _sep->handleRequest(opCtx.get(), _inMessage); + _dbResponse = _sep->handleRequest(opCtx.get(), _inMessage); // Calling reset() to wait for Eloq-Tx be committed/aborted before set _coroStatus to Empty. opCtx.reset(nullptr); + invariant(_coroStatus == CoroStatus::OnGoing); _coroStatus = CoroStatus::Empty; _serviceExecutor->ongoingCoroutineCountUpdate( _threadGroupId.load(std::memory_order_relaxed), -1); @@ -491,6 +493,11 @@ void ServiceStateMachine::_processMessage(ThreadGuard guard) { // opCtx.reset(); // _serviceContext->destoryOperationContext(&opCtx); } +} + +void ServiceStateMachine::_processResponse(ThreadGuard guard) { + invariant(!_source); + auto dbresponse = std::move(_dbResponse); // Format our response, if we have one Message& toSink = dbresponse.response; @@ -510,11 +517,12 @@ void ServiceStateMachine::_processMessage(ThreadGuard guard) { networkCounter.hitLogicalOut(toSink.size()); if (_compressorId) { + auto& compressorMgr = MessageCompressorManager::forSession(_session()); auto swm = compressorMgr.compressMessage(toSink, &_compressorId.value()); uassertStatusOK(swm.getStatus()); toSink = swm.getValue(); } - _sinkMessage(std::move(guard), std::move(toSink)); + return _sinkMessage(std::move(guard), std::move(toSink)); } else { _state.store(State::Source); @@ -551,29 +559,16 @@ void ServiceStateMachine::_runNextInGuard(ThreadGuard guard) { case State::Process: { if (!serverGlobalParams.enableCoroutine) { _processMessage(std::move(guard)); + return _processResponse(ThreadGuard(this)); } else { if (_coroStatus == CoroStatus::Empty) { MONGO_LOG(1) << "coroutine begin"; _coroStatus = CoroStatus::OnGoing; _serviceExecutor->ongoingCoroutineCountUpdate( _threadGroupId.load(std::memory_order_relaxed), 1); - _resumeTask = [ssm = this] { - Client::setCurrent(std::move(ssm->_dbClient)); - if (ssm->_migrating.load(std::memory_order_relaxed)) { - ssm->_coroResume(); - ssm->_coroYield(); - } else { - ssm->_runResumeProcess(); - bool migrating = true; - ssm->_migrating.compare_exchange_strong( - migrating, false, std::memory_order_acq_rel); - if (migrating) { - MONGO_LOG(1) - << "Migrate logical session to " - << ssm->_threadGroupId.load(std::memory_order_relaxed); - } - } - }; + // Session ownership lasts until final cleanup. Do not access the SSM + // after this call: it may finish the request and release that ownership. + _resumeTask = [ssm = this] { ssm->_runResumeProcess(); }; _coroResume = _serviceExecutor->coroutineResumeFunctor( _threadGroupId.load(std::memory_order_relaxed), _resumeTask); @@ -588,44 +583,38 @@ void ServiceStateMachine::_runNextInGuard(ThreadGuard guard) { std::allocator_arg, prealloc, NoopAllocator(), - [wssm = weak_from_this(), &guard](boost::context::continuation&& sink) { - auto ssm = wssm.lock(); - if (!ssm) { - return std::move(sink); - } - ssm->_coroYield = [ssm = ssm.get(), &sink]() { + [ssm = this, &guard](boost::context::continuation&& sink) { + // Session ownership lasts until this coroutine returns and final + // cleanup runs on the executor stack. + ssm->_coroYield = [ssm, &sink]() { MONGO_LOG(3) << "call yield"; ssm->_dbClient = Client::releaseCurrent(); sink = sink.resume(); }; - ssm->_processMessage(std::move(guard)); + try { + ssm->_processMessage(std::move(guard)); + } catch (const DBException&) { + // Keep cleanup on the executor stack, including when a request + // throws after yielding and resuming on this coroutine stack. + ssm->_coroutineException = std::current_exception(); + if (ssm->_coroStatus == CoroStatus::OnGoing) { + ssm->_coroStatus = CoroStatus::Empty; + ssm->_serviceExecutor->ongoingCoroutineCountUpdate( + ssm->_threadGroupId.load(std::memory_order_relaxed), -1); + } + } catch (const std::exception& e) { + error() << "Uncaught std::exception: " << e.what() + << ", terminating"; + quickExit(EXIT_UNCAUGHT); + } return std::move(sink); }); - - bool migrating = true; - _migrating.compare_exchange_strong( - migrating, false, std::memory_order_acq_rel); - if (migrating) { - MONGO_LOG(1) << "Migrate logical session to " - << _threadGroupId.load(std::memory_order_relaxed); - } - - // _source = - // boost::context::callcc([this, &guard](boost::context::continuation&& - // sink) { - // _coroYield = [this, &sink]() { - // MONGO_LOG(1) << "call yield"; - // _dbClient = Client::releaseCurrent(); - // sink = sink.resume(); - // }; - // _processMessage(std::move(guard)); - // return std::move(sink); - // }); } else if (_coroStatus == CoroStatus::OnGoing) { MONGO_LOG(1) << "coroutine ongoing"; _source = _source.resume(); } + return _finishCoroutineStep(); } } break; case State::EndSession: @@ -653,12 +642,53 @@ void ServiceStateMachine::_runNextInGuard(ThreadGuard guard) { void ServiceStateMachine::_runResumeProcess() { MONGO_LOG(3) << "ServiceStateMachine::_resumeRun"; + Client::setCurrent(std::move(_dbClient)); + if (_migrating.load(std::memory_order_relaxed)) { + _coroResume(); + _coroYield(); + return; + } + if (_coroStatus == CoroStatus::OnGoing) { MONGO_LOG(3) << "coroutine ongoing"; _source = _source.resume(); + return _finishCoroutineStep(); } } +void ServiceStateMachine::_finishCoroutineStep() { + bool migrating = true; + _migrating.compare_exchange_strong(migrating, false, std::memory_order_acq_rel); + if (migrating) { + MONGO_LOG(1) << "Migrate logical session to " + << _threadGroupId.load(std::memory_order_relaxed); + } + + if (_coroStatus == CoroStatus::OnGoing) { + // The request yielded. Keep the session registered until it resumes and finishes. + return; + } + invariant(!_source); + + try { + if (_coroutineException) { + std::rethrow_exception(std::exchange(_coroutineException, nullptr)); + } + // All coroutine and resume bookkeeping is complete. Response delivery can invoke + // cleanup inline, so this must be the last operation that accesses the SSM. + return _processResponse(ThreadGuard(this)); + } catch (const DBException& e) { + log() << "DBException handling request, closing client connection: " << redact(e); + } catch (const std::exception& e) { + error() << "Uncaught std::exception: " << e.what() << ", terminating"; + quickExit(EXIT_UNCAUGHT); + } + + _state.store(State::EndSession); + // This can destroy the SSM. No access to its members is allowed after final cleanup. + return _cleanupSession(ThreadGuard(this)); +} + void ServiceStateMachine::start(Ownership ownershipModel) { MONGO_LOG(1) << "ServiceStateMachine::start"; _scheduleNextWithGuard(ThreadGuard(this), @@ -779,9 +809,12 @@ void ServiceStateMachine::_terminateAndLogIfError(Status status) { void ServiceStateMachine::_cleanupSession(ThreadGuard guard) { MONGO_LOG(1) << "ServiceStateMachine::_cleanupSession"; + invariant(_coroStatus == CoroStatus::Empty); + invariant(!_source); _state.store(State::Ended); _inMessage.reset(); + _dbResponse = {}; // By ignoring the return value of Client::releaseCurrent() we destroy the session. // _dbClient is now nullptr and _dbClientPtr is invalid and should never be accessed. diff --git a/src/mongo/transport/service_state_machine.h b/src/mongo/transport/service_state_machine.h index e68db8f705..a099a68ab3 100644 --- a/src/mongo/transport/service_state_machine.h +++ b/src/mongo/transport/service_state_machine.h @@ -32,6 +32,7 @@ #include #include #include +#include #include #include "boost/optional/optional.hpp" @@ -213,12 +214,21 @@ class ServiceStateMachine : public std::enable_shared_from_this _compressorId; Message _inMessage; + DbResponse _dbResponse; AtomicWord _owned{Ownership::kUnowned}; #if MONGO_CONFIG_DEBUG_BUILD @@ -286,6 +297,9 @@ class ServiceStateMachine : public std::enable_shared_from_thisgetClient()->coroutineFunctors(); + (*coro.resumeFuncPtr)(); + (*coro.yieldFuncPtr)(); + } if (_uassertInHandler) uassert(40469, "Synthetic uassert failure", false); @@ -100,6 +114,10 @@ class MockSEP : public ServiceEntryPoint { _uassertInHandler = true; } + void setYieldInHandler() { + _yieldInHandler = true; + } + bool ranHandler() { bool ret = _ranHandler; _ranHandler = false; @@ -108,6 +126,7 @@ class MockSEP : public ServiceEntryPoint { private: bool _uassertInHandler = false; + bool _yieldInHandler = false; bool _ranHandler = false; }; @@ -242,8 +261,29 @@ class MockServiceExecutor : public ServiceExecutor { _scheduleHook = std::move(hook); } + std::function coroutineResumeFunctor(uint16_t, const Task& task) override { + return [this, task] { _resumeTask = task; }; + } + + void ongoingCoroutineCountUpdate(uint16_t, int delta) override { + _ongoingCoroutines += delta; + ASSERT_GTE(_ongoingCoroutines, 0); + } + + int ongoingCoroutines() const { + return _ongoingCoroutines; + } + + void resumeCoroutine() { + auto task = std::move(_resumeTask); + ASSERT_TRUE(static_cast(task)); + task(); + } + private: ScheduleHook _scheduleHook; + Task _resumeTask; + int _ongoingCoroutines = 0; }; class SimpleEvent { @@ -304,6 +344,20 @@ class ServiceStateMachineFixture : public unittest::Test { void runPingTest(State first, State second); void checkPingOk(); + std::weak_ptr expectFinalCleanup() { + std::weak_ptr weakSSM = _ssm; + _ssm->setCleanupHook([this, weakSSM] { + ASSERT_FALSE(haveClient()); + ASSERT_EQ(_sexec->ongoingCoroutines(), 0); + // Only the session owner remains: neither the coroutine nor its resume callback + // may hold an extra reference that masks premature cleanup. + ASSERT_EQ(weakSSM.use_count(), 1); + _ssm.reset(); + ASSERT_TRUE(weakSSM.expired()); + }); + return weakSSM; + } + MockTL* _tl; MockSEP* _sep; MockServiceExecutor* _sexec; @@ -337,6 +391,7 @@ void ServiceStateMachineFixture::checkPingOk() { TEST_F(ServiceStateMachineFixture, TestOkaySimpleCommand) { runPingTest(State::Process, State::Source); checkPingOk(); + ASSERT_EQ(_sexec->ongoingCoroutines(), 0); } TEST_F(ServiceStateMachineFixture, TestThrowHandling) { @@ -346,6 +401,79 @@ TEST_F(ServiceStateMachineFixture, TestThrowHandling) { ASSERT(_tl->getLastSunk().empty()); ASSERT_TRUE(_tl->ranSource()); ASSERT_FALSE(_tl->ranSink()); + ASSERT_EQ(_sexec->ongoingCoroutines(), 0); +} + +TEST_F(ServiceStateMachineFixture, TestThrowAfterCoroutineResumeReleasesSession) { + _sep->setYieldInHandler(); + _sep->setUassertInHandler(); + auto weakSSM = expectFinalCleanup(); + + runPingTest(State::Process, State::Process); + ASSERT_EQ(_sexec->ongoingCoroutines(), 1); + _sexec->resumeCoroutine(); + + ASSERT_FALSE(haveClient()); + ASSERT_TRUE(weakSSM.expired()); + ASSERT_EQ(_sexec->ongoingCoroutines(), 0); + ASSERT_TRUE(_tl->getLastSunk().empty()); + ASSERT_FALSE(_tl->ranSink()); +} + +TEST_F(ServiceStateMachineFixture, TestThrowBeforeCoroutineYieldReleasesSession) { + _sep->setUassertInHandler(); + auto weakSSM = expectFinalCleanup(); + + _ssm->runNext(); + ASSERT_EQ(State::Process, _ssm->state()); + _ssm->runNext(); + + ASSERT_TRUE(weakSSM.expired()); + ASSERT_FALSE(_tl->ranSink()); +} + +TEST_F(ServiceStateMachineFixture, TestSinkErrorAfterCoroutineReturnReleasesSession) { + _tl->setNextFailure(MockTL::Sink); + auto weakSSM = expectFinalCleanup(); + + _ssm->runNext(); + ASSERT_EQ(State::Process, _ssm->state()); + _ssm->runNext(); + + ASSERT_TRUE(weakSSM.expired()); + ASSERT_TRUE(_tl->ranSink()); +} + +TEST_F(ServiceStateMachineFixture, TestSinkErrorAfterCoroutineResumeReleasesSession) { + _sep->setYieldInHandler(); + _tl->setNextFailure(MockTL::Sink); + auto weakSSM = expectFinalCleanup(); + + runPingTest(State::Process, State::Process); + ASSERT_EQ(_sexec->ongoingCoroutines(), 1); + _sexec->resumeCoroutine(); + + ASSERT_TRUE(weakSSM.expired()); + ASSERT_TRUE(_tl->ranSink()); +} + +TEST_F(ServiceStateMachineFixture, TerminateWhileCoroutineSuspendedDefersCleanup) { + _sep->setYieldInHandler(); + auto weakSSM = expectFinalCleanup(); + + runPingTest(State::Process, State::Process); + // The session owner alone keeps the suspended SSM alive; the coroutine does not own itself. + ASSERT_EQ(weakSSM.use_count(), 1); + _ssm->terminate(); + ASSERT_FALSE(weakSSM.expired()); + ASSERT_EQ(State::Process, _ssm->state()); + ASSERT_EQ(_sexec->ongoingCoroutines(), 1); + + // Termination closes the socket but must leave the suspended request alive to finish. + _sexec->resumeCoroutine(); + ASSERT_TRUE(weakSSM.expired()); + ASSERT_TRUE(_tl->ranSink()); + ASSERT_TRUE(_tl->getLastSunk().empty()); } TEST_F(ServiceStateMachineFixture, TestSourceError) { diff --git a/src/mongo/transport/transport_layer_asio_test.cpp b/src/mongo/transport/transport_layer_asio_test.cpp index b9ff5580dc..4e0d2cb46a 100644 --- a/src/mongo/transport/transport_layer_asio_test.cpp +++ b/src/mongo/transport/transport_layer_asio_test.cpp @@ -46,6 +46,14 @@ namespace { class ServiceEntryPointUtil : public ServiceEntryPoint { public: + Status start() override { + return Status::OK(); + } + + transport::ServiceExecutor* getServiceExecutor() override { + return nullptr; + } + void startSession(transport::SessionHandle session) override { stdx::unique_lock lk(_mutex); _sessions.push_back(std::move(session)); @@ -161,6 +169,14 @@ TEST(TransportLayerASIO, PortZeroConnect) { class TimeoutSEP : public ServiceEntryPoint { public: + Status start() override { + return Status::OK(); + } + + transport::ServiceExecutor* getServiceExecutor() override { + return nullptr; + } + void endAllSessions(transport::Session::TagMask tags) override { MONGO_UNREACHABLE; } diff --git a/src/mongo/unittest/death_test.cpp b/src/mongo/unittest/death_test.cpp index e9def2e6e7..807b124959 100644 --- a/src/mongo/unittest/death_test.cpp +++ b/src/mongo/unittest/death_test.cpp @@ -38,6 +38,10 @@ #include #endif +#ifdef __linux__ +#include +#endif + #if defined(__APPLE__) #include #endif @@ -120,6 +124,12 @@ void DeathTestImpl::_doTest() { // process calls std::abort() or std::terminate(). const struct rlimit kNoCoreDump { 0U, 0U }; checkSyscall(setrlimit(RLIMIT_CORE, &kNoCoreDump)); +#ifdef __linux__ + // Linux ignores RLIMIT_CORE for piped core_pattern collectors (including WSL's). + // Expected deaths must not launch a crash collector or dump this large test process. + // Only this child becomes nondumpable; unexpected parent/server crashes retain diagnostics. + checkSyscall(prctl(PR_SET_DUMPABLE, 0, 0, 0, 0)); +#endif try { auto test = _makeTest(); diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript index c9414e0c28..2712581e18 100644 --- a/src/mongo/util/SConscript +++ b/src/mongo/util/SConscript @@ -310,6 +310,7 @@ env.CppUnitTest( ], LIBDEPS=[ "background_job", + "$BUILD_DIR/mongo/db/service_context", ], ) diff --git a/src/mongo/util/clock_source.cpp b/src/mongo/util/clock_source.cpp index d562089c27..4f24a0b419 100644 --- a/src/mongo/util/clock_source.cpp +++ b/src/mongo/util/clock_source.cpp @@ -35,53 +35,6 @@ namespace mongo { stdx::cv_status ClockSource::waitForConditionUntil(stdx::condition_variable& cv, stdx::unique_lock& m, Date_t deadline) { - if (_tracksSystemClock) { - return cv.wait_until(m, deadline.toSystemTimePoint()); - } - - // The rest of this function only runs during testing, when the clock source is virtualized and - // does not track the system clock. - - if (deadline <= now()) { - return stdx::cv_status::timeout; - } - - struct AlarmInfo { - stdx::mutex controlMutex; - stdx::mutex* waitMutex; - stdx::condition_variable* waitCV; - stdx::cv_status cvWaitResult = stdx::cv_status::no_timeout; - }; - auto alarmInfo = std::make_shared(); - alarmInfo->waitCV = &cv; - alarmInfo->waitMutex = m.mutex(); - const auto waiterThreadId = stdx::this_thread::get_id(); - bool invokedAlarmInline = false; - invariant(setAlarm(deadline, [alarmInfo, waiterThreadId, &invokedAlarmInline] { - stdx::lock_guard controlLk(alarmInfo->controlMutex); - alarmInfo->cvWaitResult = stdx::cv_status::timeout; - if (!alarmInfo->waitMutex) { - return; - } - if (stdx::this_thread::get_id() == waiterThreadId) { - // In NetworkInterfaceMock, setAlarm may invoke its callback immediately if the deadline - // has expired, so we detect that case and avoid self-deadlock by returning early, here. - // It is safe to set invokedAlarmInline without synchronization in this case, because it - // is exactly the case where the same thread is writing and consulting the value. - invokedAlarmInline = true; - return; - } - stdx::lock_guard waitLk(*alarmInfo->waitMutex); - alarmInfo->waitCV->notify_all(); - })); - if (!invokedAlarmInline) { - cv.wait(m); - } - m.unlock(); - stdx::lock_guard controlLk(alarmInfo->controlMutex); - m.lock(); - alarmInfo->waitMutex = nullptr; - alarmInfo->waitCV = nullptr; - return alarmInfo->cvWaitResult; + return waitForConditionUntil(cv, m, deadline); } } // namespace mongo diff --git a/src/mongo/util/clock_source.h b/src/mongo/util/clock_source.h index e38ce7c932..6135ee3a0d 100644 --- a/src/mongo/util/clock_source.h +++ b/src/mongo/util/clock_source.h @@ -28,9 +28,11 @@ #pragma once +#include #include "mongo/stdx/condition_variable.h" #include "mongo/stdx/functional.h" #include "mongo/stdx/mutex.h" +#include "mongo/stdx/thread.h" #include "mongo/util/time_support.h" namespace mongo { @@ -81,6 +83,52 @@ class ClockSource { stdx::unique_lock& m, Date_t deadline); + // Also support coroutine-aware locks/condition variables without bypassing a + // virtual clock. The same alarm protocol must be used by both kinds of waiter. + template + stdx::cv_status waitForConditionUntil(ConditionVariable& cv, + stdx::unique_lock& m, + Date_t deadline) { + if (_tracksSystemClock) + return cv.wait_until(m, deadline.toSystemTimePoint()); + if (deadline <= now()) + return stdx::cv_status::timeout; + + struct AlarmInfo { + stdx::mutex controlMutex; + Mutex* waitMutex; + ConditionVariable* waitCV; + stdx::cv_status result = stdx::cv_status::no_timeout; + }; + auto alarm = std::make_shared(); + alarm->waitCV = &cv; + alarm->waitMutex = m.mutex(); + const auto waiterThreadId = stdx::this_thread::get_id(); + bool invokedInline = false; + invariant(setAlarm(deadline, [alarm, waiterThreadId, &invokedInline] { + stdx::lock_guard control(alarm->controlMutex); + alarm->result = stdx::cv_status::timeout; + if (!alarm->waitMutex) + return; + // A mock clock may invoke an already-expired alarm in setAlarm(). + // The caller already owns waitMutex, so it must not lock it again. + if (stdx::this_thread::get_id() == waiterThreadId) { + invokedInline = true; + return; + } + stdx::lock_guard waiting(*alarm->waitMutex); + alarm->waitCV->notify_all(); + })); + if (!invokedInline) + cv.wait(m); + m.unlock(); + stdx::lock_guard control(alarm->controlMutex); + m.lock(); + alarm->waitMutex = nullptr; + alarm->waitCV = nullptr; + return alarm->result; + } + /** * Like cv.wait_until(m, deadline, pred), but uses this ClockSource instead of * stdx::chrono::system_clock to measure the passage of time. diff --git a/src/mongo/util/options_parser/options_parser_test.cpp b/src/mongo/util/options_parser/options_parser_test.cpp index edbece614e..13d893c2da 100644 --- a/src/mongo/util/options_parser/options_parser_test.cpp +++ b/src/mongo/util/options_parser/options_parser_test.cpp @@ -835,7 +835,10 @@ TEST(Style, NoGuessing) { argv.push_back("--hel"); std::map env_map; - ASSERT_NOT_OK(parser.run(testOpts, argv, env_map, &environment)); + // Eloq allows unregistered options for the Data Substrate's flag parser. An unknown + // abbreviation may pass through, but must not be guessed as the registered --help flag. + ASSERT_OK(parser.run(testOpts, argv, env_map, &environment)); + ASSERT_FALSE(environment.count("help")); } TEST(Style, LongDisguises) { diff --git a/src/mongo/util/unordered_fast_key_table.h b/src/mongo/util/unordered_fast_key_table.h index c47ffadd05..dd7c53d342 100644 --- a/src/mongo/util/unordered_fast_key_table.h +++ b/src/mongo/util/unordered_fast_key_table.h @@ -76,7 +76,7 @@ class UnorderedFastKeyTable { Entry() = default; Entry(const Entry& other) - : _used(other._used), _everUsed(other.everUsed), _curHash(other._curHash) { + : _used(other._used), _everUsed(other._everUsed), _curHash(other._curHash) { if (other.isUsed()) { new (&_data) value_type(other.getData()); } diff --git a/tests/jstests/core/set_param1.js b/tests/jstests/core/set_param1.js index 06b1365cbe..0d4edaefde 100644 --- a/tests/jstests/core/set_param1.js +++ b/tests/jstests/core/set_param1.js @@ -124,7 +124,12 @@ assert.commandWorked( db.adminCommand({"setParameter": 1, logComponentVerbosity: old.logComponentVerbosity})); var isMongos = (db.isMaster().msg === 'isdbgrid'); -if (!isMongos) { +// The standalone EloqDoc build omits MongoDB replication commands and parameters. +// Keep all generic checks above. Probe the command set, not the parameters under test, +// so a missing oplog-fetcher parameter still fails on builds that include replication. +var supportsReplication = assert.commandWorked(db.adminCommand({listCommands: 1})) + .commands.hasOwnProperty('replSetGetStatus'); +if (!isMongos && supportsReplication) { // // oplogFetcherSteadyStateMaxFetcherRestarts // From 246150a99a9e12119680f67f3db43e6683ab420e Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Mon, 21 Sep 2026 03:58:59 -0700 Subject: [PATCH 2/5] Reduce CI variants and run CMake smoke tests on EloqStore S3 Install and validate Python 3 PyYAML for IDL generation. Restrict CI to amd64 RelWithDebInfo, retain two SCons S3 backends, and use EloqStore/S3 for the single CMake job. Add S3 smoke fixture configuration, regression checks, and documentation. --- .github/workflows/ci.yml | 42 +++++++------ CMakeLists.txt | 16 +++++ cmake/MongoTests.cmake | 4 ++ cmake/tests/server_smoke.py | 25 ++++---- cmake/tests/server_smoke_config.py | 52 ++++++++++++++++ cmake/tests/test_server_smoke_config.py | 80 +++++++++++++++++++++++++ docs/how-to-compile.md | 18 +++++- 7 files changed, 207 insertions(+), 30 deletions(-) create mode 100644 cmake/tests/server_smoke_config.py create mode 100644 cmake/tests/test_server_smoke_config.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49a6e83e8b..fc91dc2a39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,12 +17,8 @@ concurrency: jobs: prime-third-party: - name: prime-third-party (${{ matrix.arch }}) - strategy: - fail-fast: false - matrix: - arch: [amd64, arm64] - runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + name: prime-third-party (amd64) + runs-on: ubuntu-24.04 container: image: eloqdata/ubuntu-dev:24.04 options: --user root @@ -46,11 +42,11 @@ jobs: uses: ./eloqdoc/.github/actions/setup-third-party with: source-dir: ${{ github.workspace }}/eloqdoc - arch: ${{ matrix.arch }} + arch: amd64 cache-scope: ubuntu-dev cmake-build-smoke: - name: CMake/Ninja single-node tests (amd64) + name: CMake/Ninja RelWithDebInfo-eloqstore_s3-amd64 needs: prime-third-party runs-on: ubuntu-24.04 timeout-minutes: 240 @@ -63,6 +59,9 @@ jobs: CMAKE_INCLUDE_PATH: /opt/eloq/third_party/include CMAKE_LIBRARY_PATH: /opt/eloq/third_party/lib LD_LIBRARY_PATH: /opt/eloq/third_party/lib:/opt/eloq/third_party/lib64 + S3_ENDPOINT: http://127.0.0.1:9900 + S3_ACCESS_KEY: rustfsadmin + S3_SECRET_KEY: rustfsadmin steps: - uses: actions/checkout@v4 with: @@ -91,10 +90,18 @@ jobs: echo "CMAKE_BUILD_PARALLEL_LEVEL=$jobs" >> "$GITHUB_ENV" echo "CTEST_PARALLEL_LEVEL=$jobs" >> "$GITHUB_ENV" + - name: Install Python 3 IDL compiler dependency + run: | + # The third-party cache contains C++ libraries, not Python code-generation dependencies. + apt-get update + apt-get install -y --no-install-recommends python3-yaml + /usr/bin/python3 -c 'import yaml' + - name: Configure top-level CMake build working-directory: eloqdoc run: | cmake -S . -B build/cmake-ci -G Ninja \ + -DPython3_EXECUTABLE=/usr/bin/python3 \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_JOB_POOLS=link_pool=1 \ -DCMAKE_JOB_POOL_LINK=link_pool \ @@ -102,8 +109,8 @@ jobs: -DELOQDOC_BUILD_TESTS=ON \ -DELOQ_THIRD_PARTY_PREFIX="$ELOQ_THIRD_PARTY_PREFIX" \ -DELOQ_THIRD_PARTY_REQUIRED=ON \ - -DWITH_DATA_STORE=ELOQDSS_ROCKSDB \ - -DWITH_LOG_STATE=ROCKSDB \ + -DWITH_DATA_STORE=ELOQDSS_ELOQSTORE \ + -DWITH_LOG_STATE=ROCKSDB_CLOUD_S3 \ -DWITH_LOG_SERVICE=ON \ -DFORK_HM_PROCESS=ON \ -DSTATISTICS=ON @@ -124,8 +131,12 @@ jobs: - name: Exercise the normal server over the MongoDB wire protocol if: ${{ !cancelled() && steps.cmake_build.outcome == 'success' }} working-directory: eloqdoc + shell: bash run: | - python3 -m pip install --target build/cmake-ci/python 'pymongo==4.8.0' + source .github/scripts/common.sh + trap 'stop_rustfs' EXIT + start_rustfs "$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" + /usr/bin/python3 -m pip install --target build/cmake-ci/python 'pymongo==4.8.0' PYTHONPATH="$PWD/build/cmake-ci/python" \ cmake --build build/cmake-ci --target eloqdoc-server-smoke @@ -138,19 +149,16 @@ jobs: strategy: fail-fast: false matrix: - build_type: [Debug, RelWithDebInfo] - arch: [amd64, arm64] + build_type: [RelWithDebInfo] + arch: [amd64] engine: - - id: rocksdb - data_store_type: ELOQDSS_ROCKSDB - log_state: ROCKSDB - id: rocks_s3 data_store_type: ELOQDSS_ROCKSDB_CLOUD_S3 log_state: ROCKSDB_CLOUD_S3 - id: eloqstore_s3 data_store_type: ELOQDSS_ELOQSTORE log_state: ROCKSDB_CLOUD_S3 - runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + runs-on: ubuntu-24.04 timeout-minutes: 240 container: image: eloqdata/ubuntu-dev:24.04 diff --git a/CMakeLists.txt b/CMakeLists.txt index 2503498923..7b98bfda13 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,22 @@ set_property(CACHE WITH_DATA_STORE PROPERTY STRINGS find_package(Python3 3.8 REQUIRED COMPONENTS Interpreter) find_package(Threads REQUIRED) +# PyYAML is needed by the IDL compiler in the selected Python 3 environment, even +# when the separately maintained C++ dependencies were restored from a cache. +execute_process( + COMMAND "${Python3_EXECUTABLE}" -c "import yaml" + RESULT_VARIABLE _eloqdoc_pyyaml_status + ERROR_VARIABLE _eloqdoc_pyyaml_error + OUTPUT_QUIET) +if(NOT _eloqdoc_pyyaml_status EQUAL 0) + message(FATAL_ERROR + "${Python3_EXECUTABLE} cannot import PyYAML (yaml), required for MongoDB IDL generation.\n" + "On Ubuntu with /usr/bin/python3, install python3-yaml. For a virtual environment, " + "install PyYAML into that environment and select it with -DPython3_EXECUTABLE=/path/to/python.\n" + "Installing PyYAML for the SCons Python 2 interpreter does not satisfy this dependency.\n" + "${_eloqdoc_pyyaml_error}") +endif() + file(GLOB_RECURSE _eloqdoc_sconscript_files CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/*SConscript") set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS diff --git a/cmake/MongoTests.cmake b/cmake/MongoTests.cmake index 03f82eb9f2..0643bf2c17 100644 --- a/cmake/MongoTests.cmake +++ b/cmake/MongoTests.cmake @@ -13,11 +13,15 @@ set_tests_properties(cmake/scons-test-selection PROPERTIES LABELS "cmake;unit" T add_test(NAME cmake/icu-data-generator COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_icu_generator.py") set_tests_properties(cmake/icu-data-generator PROPERTIES LABELS "cmake;unit" TIMEOUT 60) +add_test(NAME cmake/server-smoke-config + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_server_smoke_config.py") +set_tests_properties(cmake/server-smoke-config PROPERTIES LABELS "cmake;unit" TIMEOUT 60) # Opt-in wire-protocol regression fixture; PyMongo is an explicit test-only dependency. add_custom_target(eloqdoc-server-smoke COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/server_smoke.py" --server "$" + --data-store "${WITH_DATA_STORE}" --log-state "${WITH_LOG_STATE}" DEPENDS eloqdoc USES_TERMINAL) diff --git a/cmake/tests/server_smoke.py b/cmake/tests/server_smoke.py index 727c9643ea..944c5c938c 100644 --- a/cmake/tests/server_smoke.py +++ b/cmake/tests/server_smoke.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Exercise the real CMake server over the wire (requires pymongo>=4.6,<4.11). -Uses a private Data Substrate fixture with two worker cores and a disposable local data -store (WAL disabled). Build with ELOQDSS_ROCKSDB/ROCKSDB, not a cloud backend. Logs and -data are retained in the printed temporary directory. +Uses a private Data Substrate fixture with two worker cores. Supports local RocksDB +(WAL disabled) and EloqStore/S3 with RocksDB/S3 logging (WAL enabled). The S3 fixture +requires S3_ENDPOINT, S3_ACCESS_KEY and S3_SECRET_KEY and a running disposable S3 +service. Logs and local data are retained in the printed temporary directory. """ import argparse @@ -23,6 +24,8 @@ from pymongo.errors import AutoReconnect, OperationFailure, PyMongoError from pymongo.read_concern import ReadConcern +from server_smoke_config import substrate_config + def free_port(): with socket.socket() as sock: @@ -53,6 +56,8 @@ def command_fails(database, command, *, code=None, message): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--server", required=True, type=Path) + parser.add_argument("--data-store", default="ELOQDSS_ROCKSDB") + parser.add_argument("--log-state", default="ROCKSDB") args = parser.parse_args() server = args.server.resolve() # Keep every fixture child, including auxiliary services, within the user's CPU limit. @@ -63,14 +68,12 @@ def main(): (root / "db").mkdir() port, tx_port = free_port(), free_port() config = root / "data_substrate.cnf" - config.write_text( - "[local]\ncore_number=2\nnode_memory_limit_mb=512\n" - "enable_data_store=true\nenable_wal=false\nenable_mvcc=true\n" - "event_dispatcher_num=1\nbind_all=false\n" - f"tx_ip=127.0.0.1\ntx_port={tx_port}\nhm_port={free_port()}\n" - f"eloq_data_path={root / 'eloq'}\nlog_service_data_path={root / 'log'}\n" - f"[cluster]\ntx_ip_port_list=127.0.0.1:{tx_port}\n", - encoding="utf-8") + try: + config_text = substrate_config(root, tx_port, free_port(), args.data_store, + args.log_state, os.environ) + except ValueError as exc: + parser.error(str(exc)) + config.write_text(config_text, encoding="utf-8") server_config = root / "eloqdoc.yaml" # JSON is valid YAML and avoids adding another test-client dependency. server_config.write_text(json.dumps({ diff --git a/cmake/tests/server_smoke_config.py b/cmake/tests/server_smoke_config.py new file mode 100644 index 0000000000..b6829264b9 --- /dev/null +++ b/cmake/tests/server_smoke_config.py @@ -0,0 +1,52 @@ +"""Data Substrate configuration for the Mongo-facing server smoke fixture.""" + +import uuid + + +def substrate_config(root, tx_port, hm_port, data_store, log_state, environment): + cloud = (data_store, log_state) == ("ELOQDSS_ELOQSTORE", "ROCKSDB_CLOUD_S3") + local = data_store in ("ROCKSDB", "ELOQDSS_ROCKSDB") and log_state == "ROCKSDB" + if not (cloud or local): + raise ValueError(f"Unsupported smoke-test backend: {data_store}/{log_state}") + + cloud_local, cloud_store = "", "" + if cloud: + required = ("S3_ENDPOINT", "S3_ACCESS_KEY", "S3_SECRET_KEY") + missing = [key for key in required if not environment.get(key)] + if missing: + raise ValueError("EloqStore/S3 smoke test requires " + ", ".join(missing)) + endpoint = environment["S3_ENDPOINT"].rstrip("/") + access_key, secret_key = environment["S3_ACCESS_KEY"], environment["S3_SECRET_KEY"] + # A fresh bucket isolates each invocation, including retries. CI owns the disposable + # S3 service; the smoke test never removes buckets from an externally supplied service. + bucket = "eloqdoc-smoke-" + uuid.uuid4().hex + cloud_local = ( + "checkpoint_interval=10\nlogserver_snapshot_interval=60\n" + "txlog_rocksdb_cloud_bucket_prefix=\n" + f"txlog_rocksdb_cloud_bucket_name={bucket}\n" + "txlog_rocksdb_cloud_object_path=txlog\n" + f"txlog_rocksdb_cloud_s3_endpoint_url={endpoint}\n" + f"txlog_rocksdb_cloud_object_store_service_url={endpoint}/{bucket}/txlog\n" + "txlog_rocksdb_cloud_sst_file_cache_size=1GB\n") + cloud_store = ( + "[store]\n" + f"aws_access_key_id={access_key}\naws_secret_key={secret_key}\n" + "eloq_store_buffer_pool_size=500MB\neloq_store_local_space_limit=2GB\n" + "eloq_store_manifest_limit=1048576\neloq_store_pages_per_file_shift=8\n" + "eloq_store_cloud_provider=aws\n" + f"eloq_store_cloud_endpoint={endpoint}\n" + f"eloq_store_cloud_store_path={bucket}/eloqstore\n" + f"eloq_store_cloud_access_key={access_key}\n" + f"eloq_store_cloud_secret_key={secret_key}\n" + "eloq_store_cloud_verify_ssl=false\neloq_store_cloud_request_threads=2\n") + + return ( + "[local]\ncore_number=2\n" + f"node_memory_limit_mb={2048 if cloud else 512}\n" + f"enable_data_store=true\nenable_wal={'true' if cloud else 'false'}\n" + "enable_mvcc=true\nevent_dispatcher_num=1\nbind_all=false\n" + f"tx_ip=127.0.0.1\ntx_port={tx_port}\nhm_port={hm_port}\n" + f"eloq_data_path={root / 'eloq'}\nlog_service_data_path={root / 'log'}\n" + + cloud_local + + f"[cluster]\ntx_ip_port_list=127.0.0.1:{tx_port}\n" + + cloud_store) diff --git a/cmake/tests/test_server_smoke_config.py b/cmake/tests/test_server_smoke_config.py new file mode 100644 index 0000000000..005fb96086 --- /dev/null +++ b/cmake/tests/test_server_smoke_config.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Check smoke-fixture backend selection without a server or cloud service.""" + +import configparser +from pathlib import Path +import unittest + +from server_smoke_config import substrate_config + + +class ServerSmokeConfigTest(unittest.TestCase): + environment = {"S3_ENDPOINT": "http://127.0.0.1:9900/", + "S3_ACCESS_KEY": "test-access", "S3_SECRET_KEY": "test-secret"} + + def config(self, data_store="ELOQDSS_ELOQSTORE", log_state="ROCKSDB_CLOUD_S3", + environment=None): + config = configparser.ConfigParser(interpolation=None) + config.read_string(substrate_config( + Path("/tmp/smoke fixture"), 12345, 12346, data_store, log_state, + self.environment if environment is None else environment)) + return config + + def test_eloqstore_s3_uses_same_bucket_and_credentials_for_data_and_log(self): + config = self.config() + local, store = config["local"], config["store"] + bucket = local["txlog_rocksdb_cloud_bucket_name"] + self.assertRegex(bucket, r"^eloqdoc-smoke-[0-9a-f]{32}$") + self.assertTrue(local.getboolean("enable_wal")) + self.assertEqual("", local["txlog_rocksdb_cloud_bucket_prefix"]) + self.assertEqual("txlog", local["txlog_rocksdb_cloud_object_path"]) + self.assertEqual(f"http://127.0.0.1:9900/{bucket}/txlog", + local["txlog_rocksdb_cloud_object_store_service_url"]) + self.assertEqual(local["txlog_rocksdb_cloud_s3_endpoint_url"], + store["eloq_store_cloud_endpoint"]) + self.assertEqual(f"{bucket}/eloqstore", store["eloq_store_cloud_store_path"]) + self.assertEqual("aws", store["eloq_store_cloud_provider"]) + self.assertEqual("test-access", store["aws_access_key_id"]) + self.assertEqual(store["aws_access_key_id"], store["eloq_store_cloud_access_key"]) + self.assertEqual("test-secret", store["aws_secret_key"]) + self.assertEqual(store["aws_secret_key"], store["eloq_store_cloud_secret_key"]) + + def test_cloud_invocations_are_isolated(self): + self.assertNotEqual(self.config()["store"]["eloq_store_cloud_store_path"], + self.config()["store"]["eloq_store_cloud_store_path"]) + + def test_missing_s3_configuration_is_rejected(self): + for key in self.environment: + with self.subTest(key=key): + environment = dict(self.environment, **{key: ""}) + with self.assertRaisesRegex(ValueError, key): + self.config(environment=environment) + + def test_local_rocksdb_needs_no_cloud_service(self): + for data_store in ("ROCKSDB", "ELOQDSS_ROCKSDB"): + with self.subTest(data_store=data_store): + config = self.config(data_store, "ROCKSDB", environment={}) + self.assertFalse(config["local"].getboolean("enable_wal")) + self.assertNotIn("store", config) + self.assertFalse(any("cloud" in key for key in config["local"])) + + def test_unsupported_backend_does_not_silently_use_local_rocksdb(self): + for data_store, log_state in (("ELOQDSS_ELOQSTORE", "ROCKSDB"), + ("ELOQDSS_ROCKSDB", "ROCKSDB_CLOUD_S3"), + ("ELOQDSS_ROCKSDB_CLOUD_S3", "ROCKSDB_CLOUD_S3")): + with self.subTest(data_store=data_store, log_state=log_state): + with self.assertRaisesRegex(ValueError, "Unsupported smoke-test backend"): + self.config(data_store, log_state) + + def test_private_paths_and_ports_are_retained(self): + config = self.config() + self.assertEqual("/tmp/smoke fixture/eloq", config["local"]["eloq_data_path"]) + self.assertEqual("/tmp/smoke fixture/log", config["local"]["log_service_data_path"]) + self.assertEqual(2, config["local"].getint("core_number")) + self.assertEqual(12345, config["local"].getint("tx_port")) + self.assertEqual(12346, config["local"].getint("hm_port")) + self.assertEqual("127.0.0.1:12345", config["cluster"]["tx_ip_port_list"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/how-to-compile.md b/docs/how-to-compile.md index 4013257d40..32447faef1 100644 --- a/docs/how-to-compile.md +++ b/docs/how-to-compile.md @@ -49,6 +49,12 @@ with the historical SCons requirements, and the Data Substrate dependency prefix distribution, install equivalent packages and build the dependencies described by `src/mongo/db/modules/eloq/data_substrate/scripts/third_party/install-ubuntu2404.sh`. +The CMake IDL compiler requires PyYAML in the Python 3 interpreter selected by CMake. +For Ubuntu's `/usr/bin/python3`, install it with `sudo apt-get install python3-yaml` +(the dependency script already does this). For a virtual environment, install `PyYAML` +there and pass `-DPython3_EXECUTABLE=/path/to/venv/bin/python` to CMake. Python 2's +SCons packages and the cached C++ dependency prefix do not provide this Python 3 module. + By default the Data Substrate dependencies are installed at: ```text @@ -256,8 +262,10 @@ its generic parameter checks and runs its oplog-fetcher checks only when the ser replication commands. This preserves those assertions for SCons without restoring replication to the CMake server or excluding the whole test. -The self-contained server smoke test requires `ELOQDSS_ROCKSDB` or `ROCKSDB`. It starts an -authenticated server with a temporary local store, two Substrate worker cores, and WAL disabled. +The server smoke test uses the build's selected storage/log backend. It supports local +`ELOQDSS_ROCKSDB` (or `ROCKSDB`) with `ROCKSDB` logging and WAL disabled, or +`ELOQDSS_ELOQSTORE` with `ROCKSDB_CLOUD_S3` logging and WAL enabled. Both start an +authenticated server with two Substrate worker cores and temporary local storage. EloqDoc's catalog requires a data-store handler even for a disposable fixture. The test checks CRUD, ICU 57.1 collation metadata and lookup, local aggregation, JavaScript, background/TTL indexes, sessions and transaction commit/abort, @@ -274,6 +282,12 @@ PYTHONPATH="$PWD/build/cmake/python" \ cmake --build build/cmake --target eloqdoc-server-smoke -j8 ``` +For EloqStore/S3, first start a disposable S3-compatible service and export `S3_ENDPOINT`, +`S3_ACCESS_KEY`, and `S3_SECRET_KEY`. The fixture uses a new bucket on each run and retains +it for inspection; stop/remove your disposable service afterward. CI starts and stops its own +RustFS instance. The CMake CI job tests only amd64/RelWithDebInfo/EloqStore-S3; the SCons CI +jobs cover amd64/RelWithDebInfo with RocksDB-S3 and EloqStore-S3. + The fixture is restricted to at most eight CPUs and retains its logs and data in the temporary directory printed at startup. It tests Mongo-facing behavior, not Data Substrate/data-store persistence or recovery. Use `--option=value` for server options with values, as in the existing From adcb16aee41bd10a83387b8e3ae8ad5e48d62a11 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Mon, 21 Sep 2026 10:59:31 -0700 Subject: [PATCH 3/5] Preserve CMake smoke failure logs and report shutdown exit status Report nonzero exits and POSIX signals separately from shutdown timeouts before fixture cleanup. Print bounded log tails, preserve full fixture logs without configs or data, and upload failed CI diagnostics. Add regression checks for exit handling and artifact collection. --- .github/workflows/ci.yml | 12 ++ cmake/MongoTests.cmake | 4 + cmake/tests/server_smoke.py | 12 +- cmake/tests/server_smoke_diagnostics.py | 72 +++++++++++ cmake/tests/test_server_smoke_diagnostics.py | 120 +++++++++++++++++++ docs/how-to-compile.md | 6 + 6 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 cmake/tests/server_smoke_diagnostics.py create mode 100644 cmake/tests/test_server_smoke_diagnostics.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc91dc2a39..fe673e35e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,6 +129,7 @@ jobs: --output-on-failure - name: Exercise the normal server over the MongoDB wire protocol + id: cmake_smoke if: ${{ !cancelled() && steps.cmake_build.outcome == 'success' }} working-directory: eloqdoc shell: bash @@ -140,6 +141,17 @@ jobs: PYTHONPATH="$PWD/build/cmake-ci/python" \ cmake --build build/cmake-ci --target eloqdoc-server-smoke + - name: Preserve failed CMake smoke-test logs + if: ${{ failure() && steps.cmake_smoke.outcome == 'failure' }} + uses: actions/upload-artifact@v4 + with: + name: cmake-smoke-eloqstore-s3-amd64-${{ github.run_attempt }} + path: | + eloqdoc/build/cmake-ci/smoke-diagnostics/ + /tmp/rustfs.log + if-no-files-found: warn + retention-days: 7 + ent-ci: needs: prime-third-party concurrency: diff --git a/cmake/MongoTests.cmake b/cmake/MongoTests.cmake index 0643bf2c17..635ec8b242 100644 --- a/cmake/MongoTests.cmake +++ b/cmake/MongoTests.cmake @@ -16,12 +16,16 @@ set_tests_properties(cmake/icu-data-generator PROPERTIES LABELS "cmake;unit" TIM add_test(NAME cmake/server-smoke-config COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_server_smoke_config.py") set_tests_properties(cmake/server-smoke-config PROPERTIES LABELS "cmake;unit" TIMEOUT 60) +add_test(NAME cmake/server-smoke-diagnostics + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_server_smoke_diagnostics.py") +set_tests_properties(cmake/server-smoke-diagnostics PROPERTIES LABELS "cmake;unit" TIMEOUT 60) # Opt-in wire-protocol regression fixture; PyMongo is an explicit test-only dependency. add_custom_target(eloqdoc-server-smoke COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/server_smoke.py" --server "$" --data-store "${WITH_DATA_STORE}" --log-state "${WITH_LOG_STATE}" + --diagnostics-dir "${CMAKE_CURRENT_BINARY_DIR}/smoke-diagnostics" DEPENDS eloqdoc USES_TERMINAL) diff --git a/cmake/tests/server_smoke.py b/cmake/tests/server_smoke.py index 944c5c938c..d4b22595e7 100644 --- a/cmake/tests/server_smoke.py +++ b/cmake/tests/server_smoke.py @@ -25,6 +25,7 @@ from pymongo.read_concern import ReadConcern from server_smoke_config import substrate_config +from server_smoke_diagnostics import describe_exit, report_failure, wait_for_clean_shutdown def free_port(): @@ -58,6 +59,8 @@ def main(): parser.add_argument("--server", required=True, type=Path) parser.add_argument("--data-store", default="ELOQDSS_ROCKSDB") parser.add_argument("--log-state", default="ROCKSDB") + parser.add_argument("--diagnostics-dir", type=Path, + help="Copy failure logs here for CI artifact collection") args = parser.parse_args() server = args.server.resolve() # Keep every fixture child, including auxiliary services, within the user's CPU limit. @@ -105,7 +108,8 @@ def main(): try: last_connection_error = [] def ready(): - assert process.poll() is None, "server exited; see " + str(root / "server.log") + returncode = process.poll() + assert returncode is None, "server exited: " + describe_exit(returncode) try: return anonymous.admin.command("ping")["ok"] == 1 except PyMongoError as exc: @@ -269,10 +273,14 @@ def sleep_is_active(): authenticated.admin.command("shutdown", force=True) except AutoReconnect: pass - assert process.wait(timeout=30) == 0 + wait_for_clean_shutdown(process, timeout=30) assert sleep_finished.wait(5), "sleep command did not finish during shutdown" assert not sleep_errors, sleep_errors print("PASS clean shutdown with an active command", flush=True) + except BaseException as exc: + # Record the real exit status and logs before fixture cleanup can send signals. + report_failure(root, process.poll(), exc, args.diagnostics_dir) + raise finally: anonymous.close() if authenticated: diff --git a/cmake/tests/server_smoke_diagnostics.py b/cmake/tests/server_smoke_diagnostics.py new file mode 100644 index 0000000000..2e239e2f8a --- /dev/null +++ b/cmake/tests/server_smoke_diagnostics.py @@ -0,0 +1,72 @@ +"""Failure diagnostics for the Mongo-facing smoke test (standard library only).""" + +import shutil +import signal +import subprocess +import sys + + +def describe_exit(returncode): + if returncode is None: + return "still running" + description = f"exit code {returncode}" + if returncode < 0: + try: + description += f" ({signal.Signals(-returncode).name})" + except ValueError: + description += f" (signal {-returncode})" + return description + + +def wait_for_clean_shutdown(process, timeout=30): + try: + returncode = process.wait(timeout=timeout) + except subprocess.TimeoutExpired as exc: + raise AssertionError(f"server shutdown timed out after {timeout}s") from exc + if returncode != 0: + raise AssertionError("server shutdown failed: " + describe_exit(returncode)) + + +def report_failure(root, returncode, error, diagnostics_dir=None): + """Print bounded log tails and preserve full logs, without masking the original failure.""" + summary = (f"Fixture: {root}\n" + f"Failure: {type(error).__name__}: {error}\n" + f"Server status before fixture cleanup: {describe_exit(returncode)}\n") + print(summary, file=sys.stderr, flush=True) + destination = None + if diagnostics_dir is not None: + try: + destination = diagnostics_dir / root.name + destination.mkdir(parents=True, exist_ok=True) + (destination / "failure.txt").write_text(summary, encoding="utf-8") + except OSError as exc: + print(f"Could not save failure summary: {exc}", file=sys.stderr, flush=True) + destination = None + + try: + # Include Mongo output, rejected-option logs, and timestamped glog files. Do not + # upload data directories, core dumps, credential-bearing configs, or symlinks. + logs = sorted(path for path in root.iterdir() + if not path.is_symlink() and path.is_file() + and (path.name.endswith(".log") or ".log." in path.name)) + except OSError as exc: + print(f"Could not list fixture logs: {exc}", file=sys.stderr, flush=True) + return + for path in logs: + try: + # Limit console output, but keep the complete file in the artifact. + with path.open("rb") as log: + log.seek(0, 2) + log.seek(max(0, log.tell() - 64 * 1024)) + tail = log.read().decode("utf-8", errors="replace").splitlines()[-200:] + print(f"--- {path.name} (last up to 200 lines / 64 KiB) ---\n" + "\n".join(tail), + file=sys.stderr, flush=True) + except OSError as exc: + print(f"Could not read {path.name}: {exc}", file=sys.stderr, flush=True) + if destination is not None: + try: + shutil.copyfile(path, destination / path.name) + except OSError as exc: + print(f"Could not preserve {path.name}: {exc}", file=sys.stderr, flush=True) + if destination is not None: + print(f"Failure diagnostics: {destination}", file=sys.stderr, flush=True) diff --git a/cmake/tests/test_server_smoke_diagnostics.py b/cmake/tests/test_server_smoke_diagnostics.py new file mode 100644 index 0000000000..bc4fd3b522 --- /dev/null +++ b/cmake/tests/test_server_smoke_diagnostics.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Check smoke-test failure reporting with real child exits and disposable log files.""" + +import contextlib +import io +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +from server_smoke_diagnostics import describe_exit, report_failure, wait_for_clean_shutdown + + +class SmokeDiagnosticsTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="eloqdoc-smoke-diagnostics-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) / "fixture" + self.root.mkdir() + self.artifacts = Path(self.temp.name) / "artifacts" + self.output = io.StringIO() + + def report(self, returncode, error): + with contextlib.redirect_stderr(self.output): + report_failure(self.root, returncode, error, self.artifacts) + return self.output.getvalue() + + def test_clean_exit_is_required(self): + with subprocess.Popen([sys.executable, "-c", "pass"]) as process: + wait_for_clean_shutdown(process, timeout=5) + self.assertEqual(0, process.returncode) + + def test_nonzero_exit_is_reported_and_full_logs_are_preserved(self): + content = "early log context\n" + "later log context\n" * 250 + "shutdown failure\n" + (self.root / "server.log").write_text(content) + with subprocess.Popen([sys.executable, "-c", "raise SystemExit(23)"]) as process: + with self.assertRaisesRegex(AssertionError, "shutdown failed: exit code 23") as failure: + wait_for_clean_shutdown(process, timeout=5) + output = self.report(process.returncode, failure.exception) + self.assertIn("Server status before fixture cleanup: exit code 23", output) + self.assertIn("shutdown failure", output) + self.assertNotIn("early log context", output) + destination = self.artifacts / self.root.name + self.assertEqual(content, (destination / "server.log").read_text()) + self.assertIn("exit code 23", (destination / "failure.txt").read_text()) + + @unittest.skipUnless(sys.platform != "win32", "POSIX signal exit codes") + def test_signal_exit_has_number_and_name(self): + # SIGTERM exercises signal reporting without generating a core dump. + with subprocess.Popen([sys.executable, "-c", + "import os, signal; os.kill(os.getpid(), signal.SIGTERM)"]) as process: + with self.assertRaisesRegex(AssertionError, r"exit code -15 \(SIGTERM\)"): + wait_for_clean_shutdown(process, timeout=5) + self.assertEqual(-signal.SIGTERM, process.returncode) + self.assertEqual("exit code -11 (SIGSEGV)", describe_exit(-signal.SIGSEGV)) + self.assertEqual("exit code -999 (signal 999)", describe_exit(-999)) + + def test_timeout_is_not_reported_as_a_crash(self): + process = mock.Mock() + process.wait.side_effect = subprocess.TimeoutExpired("server", 30) + with self.assertRaisesRegex(AssertionError, "shutdown timed out after 30s") as failure: + wait_for_clean_shutdown(process, timeout=30) + self.assertIsInstance(failure.exception.__cause__, subprocess.TimeoutExpired) + output = self.report(None, failure.exception) + self.assertIn("Server status before fixture cleanup: still running", output) + self.assertNotIn("SIGTERM", output) + process.terminate.assert_not_called() + process.kill.assert_not_called() + + def test_artifacts_exclude_configs_data_core_files_and_symlinks(self): + included = ("server.log", "replSet.log", "eloqdb.log.FATAL.20260921.123", + "host_manager.log.INFO.20260921.456") + for name in included: + (self.root / name).write_text("test log") + for name in ("data_substrate.cnf", "eloqdoc.yaml", "core.123"): + (self.root / name).write_text("must not upload") + (self.root / "db").mkdir() + (self.root / "db" / "nested.log").write_text("data must not upload") + (self.root / "linked.log").symlink_to(self.root / "data_substrate.cnf") + self.report(-11, AssertionError("crashed")) + self.assertEqual(set(included) | {"failure.txt"}, + {path.name for path in (self.artifacts / self.root.name).iterdir()}) + self.assertNotIn("must not upload", self.output.getvalue()) + + def test_huge_line_and_non_utf8_log_do_not_break_diagnostics(self): + content = b"x" * (128 * 1024) + b"\xfflast line\n" + (self.root / "server.log").write_bytes(content) + output = self.report(1, AssertionError("failure")) + self.assertIn("last line", output) + self.assertLess(len(output), 66 * 1024) + self.assertEqual(content, (self.artifacts / self.root.name / "server.log").read_bytes()) + + def test_unwritable_artifact_destination_still_prints_logs(self): + self.artifacts.write_text("a file blocks directory creation") + (self.root / "server.log").write_text("shutdown stack trace") + output = self.report(1, AssertionError("original failure")) + self.assertIn("Could not save failure summary", output) + self.assertIn("original failure", output) + self.assertIn("shutdown stack trace", output) + + def test_copy_failure_does_not_mask_original_failure(self): + (self.root / "server.log").write_text("shutdown stack trace") + with mock.patch("server_smoke_diagnostics.shutil.copyfile", side_effect=OSError("disk full")): + output = self.report(1, AssertionError("original failure")) + self.assertIn("Could not preserve server.log: disk full", output) + self.assertIn("original failure", output) + self.assertIn("shutdown stack trace", output) + + def test_missing_logs_do_not_mask_original_failure(self): + self.root.rmdir() + output = self.report(1, AssertionError("original failure")) + self.assertIn("Could not list fixture logs", output) + self.assertIn("original failure", output) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/how-to-compile.md b/docs/how-to-compile.md index 32447faef1..3586f10b8f 100644 --- a/docs/how-to-compile.md +++ b/docs/how-to-compile.md @@ -293,6 +293,12 @@ directory printed at startup. It tests Mongo-facing behavior, not Data Substrate persistence or recovery. Use `--option=value` for server options with values, as in the existing SCons launch scripts: the gflags pass can reorder separate argument values. +On a runtime failure, the smoke test reports the server's exit code (and signal name when +applicable) before cleanup, prints bounded log tails, and copies full fixture logs plus a failure +summary to `build/cmake/smoke-diagnostics/`. CI uploads these diagnostics and the RustFS log +as a seven-day artifact. Data directories and configuration files are not uploaded. A shutdown +timeout remains distinct from a nonzero exit, and clean shutdown still requires exit code zero. + ## 5. Inspect the selected MongoDB graph The source list is derived statically from MongoDB's fine-grained `SConscript` library metadata; From 99236416fe2bd4c40cd562f07a70a439a5dedcfa Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Mon, 21 Sep 2026 12:58:45 -0700 Subject: [PATCH 4/5] Drain TTL monitor before storage shutdown Wake the TTL worker promptly and drain active passes before storage teardown. Use a coroutine-aware wait so command-driven shutdown does not block the Substrate worker needed to finish a TTL request. Run sleeping-worker and single-worker active-pass shutdown regressions in the existing CMake server smoke target. Keep the production fix shared with SCons. --- cmake/MongoTests.cmake | 6 ++ cmake/tests/server_smoke.py | 30 ++++++++- cmake/tests/server_smoke_config.py | 4 +- cmake/tests/test_server_smoke_config.py | 11 +++- docs/how-to-compile.md | 9 ++- src/mongo/db/db.cpp | 4 ++ src/mongo/db/ttl.cpp | 81 +++++++++++++++++++++++-- src/mongo/db/ttl.h | 2 + 8 files changed, 134 insertions(+), 13 deletions(-) diff --git a/cmake/MongoTests.cmake b/cmake/MongoTests.cmake index 635ec8b242..33e516e506 100644 --- a/cmake/MongoTests.cmake +++ b/cmake/MongoTests.cmake @@ -26,6 +26,12 @@ add_custom_target(eloqdoc-server-smoke --server "$" --data-store "${WITH_DATA_STORE}" --log-state "${WITH_LOG_STATE}" --diagnostics-dir "${CMAKE_CURRENT_BINARY_DIR}/smoke-diagnostics" + --ttl-shutdown-mode sleeping + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/server_smoke.py" + --server "$" + --data-store "${WITH_DATA_STORE}" --log-state "${WITH_LOG_STATE}" + --diagnostics-dir "${CMAKE_CURRENT_BINARY_DIR}/smoke-diagnostics" + --ttl-shutdown-mode active DEPENDS eloqdoc USES_TERMINAL) diff --git a/cmake/tests/server_smoke.py b/cmake/tests/server_smoke.py index d4b22595e7..c47958dbd2 100644 --- a/cmake/tests/server_smoke.py +++ b/cmake/tests/server_smoke.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Exercise the real CMake server over the wire (requires pymongo>=4.6,<4.11). -Uses a private Data Substrate fixture with two worker cores. Supports local RocksDB +Uses a private Data Substrate fixture with one or two worker cores. Supports local RocksDB (WAL disabled) and EloqStore/S3 with RocksDB/S3 logging (WAL enabled). The S3 fixture requires S3_ENDPOINT, S3_ACCESS_KEY and S3_SECRET_KEY and a running disposable S3 service. Logs and local data are retained in the printed temporary directory. @@ -59,6 +59,7 @@ def main(): parser.add_argument("--server", required=True, type=Path) parser.add_argument("--data-store", default="ELOQDSS_ROCKSDB") parser.add_argument("--log-state", default="ROCKSDB") + parser.add_argument("--ttl-shutdown-mode", choices=("sleeping", "active"), default="sleeping") parser.add_argument("--diagnostics-dir", type=Path, help="Copy failure logs here for CI artifact collection") args = parser.parse_args() @@ -73,7 +74,10 @@ def main(): config = root / "data_substrate.cnf" try: config_text = substrate_config(root, tx_port, free_port(), args.data_store, - args.log_state, os.environ) + args.log_state, os.environ, + # One worker makes a blocking shutdown wait deadlock + # deterministically if TTL still needs that worker. + core_number=1 if args.ttl_shutdown_mode == "active" else 2) except ValueError as exc: parser.error(str(exc)) config.write_text(config_text, encoding="utf-8") @@ -244,6 +248,21 @@ def ready(): assert db.records_view.count_documents({}) == 9 assert db.records.find_one({"k": 0})["v"] == 20 print("PASS unsafe online catalog restart rejected without affecting data", flush=True) + def server_logged(message): + return message in (root / "server.log").read_text(encoding="utf-8", errors="replace") + if args.ttl_shutdown_mode == "sleeping": + # Confirm the worker enters a wait much longer than the shutdown timeout. + # Merely joining an uninterruptible sleep must fail this regression check. + authenticated.admin.command({"setParameter": 1, + "logComponentVerbosity": {"index": {"verbosity": 1}}, + "ttlMonitorSleepSecs": 600}) + eventually(lambda: server_logged("TTL monitor waiting for next pass: 600 seconds"), 10) + print("PASS TTL worker entered its long sleep before shutdown", flush=True) + else: + authenticated.admin.command({"configureFailPoint": "hangTTLMonitorBeforeStorageAccess", + "mode": "alwaysOn"}) + eventually(lambda: server_logged("TTL monitor paused before storage access"), 10) + print("PASS active TTL pass held before shutdown", flush=True) # The network integration suites can leave a long-running command behind. Shutdown # must interrupt it, not wait for its normal completion or require SIGTERM. sleep_finished = threading.Event() @@ -276,6 +295,13 @@ def sleep_is_active(): wait_for_clean_shutdown(process, timeout=30) assert sleep_finished.wait(5), "sleep command did not finish during shutdown" assert not sleep_errors, sleep_errors + shutdown_log = (root / "server.log").read_text(encoding="utf-8", errors="replace") + ttl_stopped = shutdown_log.index("TTL monitor stopped") + storage_stopped = shutdown_log.index("EloqKVEngine::cleanShutdown") + assert ttl_stopped < storage_stopped, "storage teardown ran before TTL completed" + if args.ttl_shutdown_mode == "active": + assert shutdown_log.index("TTL monitor resuming storage access") < ttl_stopped + print("PASS TTL worker drained before storage teardown", flush=True) print("PASS clean shutdown with an active command", flush=True) except BaseException as exc: # Record the real exit status and logs before fixture cleanup can send signals. diff --git a/cmake/tests/server_smoke_config.py b/cmake/tests/server_smoke_config.py index b6829264b9..d060e139a9 100644 --- a/cmake/tests/server_smoke_config.py +++ b/cmake/tests/server_smoke_config.py @@ -3,7 +3,7 @@ import uuid -def substrate_config(root, tx_port, hm_port, data_store, log_state, environment): +def substrate_config(root, tx_port, hm_port, data_store, log_state, environment, *, core_number=2): cloud = (data_store, log_state) == ("ELOQDSS_ELOQSTORE", "ROCKSDB_CLOUD_S3") local = data_store in ("ROCKSDB", "ELOQDSS_ROCKSDB") and log_state == "ROCKSDB" if not (cloud or local): @@ -41,7 +41,7 @@ def substrate_config(root, tx_port, hm_port, data_store, log_state, environment) "eloq_store_cloud_verify_ssl=false\neloq_store_cloud_request_threads=2\n") return ( - "[local]\ncore_number=2\n" + f"[local]\ncore_number={core_number}\n" f"node_memory_limit_mb={2048 if cloud else 512}\n" f"enable_data_store=true\nenable_wal={'true' if cloud else 'false'}\n" "enable_mvcc=true\nevent_dispatcher_num=1\nbind_all=false\n" diff --git a/cmake/tests/test_server_smoke_config.py b/cmake/tests/test_server_smoke_config.py index 005fb96086..b0b017fb92 100644 --- a/cmake/tests/test_server_smoke_config.py +++ b/cmake/tests/test_server_smoke_config.py @@ -13,11 +13,11 @@ class ServerSmokeConfigTest(unittest.TestCase): "S3_ACCESS_KEY": "test-access", "S3_SECRET_KEY": "test-secret"} def config(self, data_store="ELOQDSS_ELOQSTORE", log_state="ROCKSDB_CLOUD_S3", - environment=None): + environment=None, core_number=2): config = configparser.ConfigParser(interpolation=None) config.read_string(substrate_config( Path("/tmp/smoke fixture"), 12345, 12346, data_store, log_state, - self.environment if environment is None else environment)) + self.environment if environment is None else environment, core_number=core_number)) return config def test_eloqstore_s3_uses_same_bucket_and_credentials_for_data_and_log(self): @@ -75,6 +75,13 @@ def test_private_paths_and_ports_are_retained(self): self.assertEqual(12346, config["local"].getint("hm_port")) self.assertEqual("127.0.0.1:12345", config["cluster"]["tx_ip_port_list"]) + def test_active_shutdown_can_use_a_single_worker(self): + for data_store, log_state in (("ELOQDSS_ROCKSDB", "ROCKSDB"), + ("ELOQDSS_ELOQSTORE", "ROCKSDB_CLOUD_S3")): + with self.subTest(data_store=data_store): + self.assertEqual(1, self.config(data_store, log_state, core_number=1)[ + "local"].getint("core_number")) + if __name__ == "__main__": unittest.main() diff --git a/docs/how-to-compile.md b/docs/how-to-compile.md index 3586f10b8f..f7a76a0961 100644 --- a/docs/how-to-compile.md +++ b/docs/how-to-compile.md @@ -265,7 +265,7 @@ to the CMake server or excluding the whole test. The server smoke test uses the build's selected storage/log backend. It supports local `ELOQDSS_ROCKSDB` (or `ROCKSDB`) with `ROCKSDB` logging and WAL disabled, or `ELOQDSS_ELOQSTORE` with `ROCKSDB_CLOUD_S3` logging and WAL enabled. Both start an -authenticated server with two Substrate worker cores and temporary local storage. +authenticated server with temporary local storage and at most two Substrate worker cores. EloqDoc's catalog requires a data-store handler even for a disposable fixture. The test checks CRUD, ICU 57.1 collation metadata and lookup, local aggregation, JavaScript, background/TTL indexes, sessions and transaction commit/abort, @@ -299,6 +299,13 @@ summary to `build/cmake/smoke-diagnostics/`. CI uploads these diagnostics and th as a seven-day artifact. Data directories and configuration files are not uploaded. A shutdown timeout remains distinct from a nonzero exit, and clean shutdown still requires exit code zero. +The smoke target runs twice to verify TTL shutdown ordering: once with the worker in a +600-second sleep (which shutdown must wake promptly), and once with an active pass paused +before storage access. Both require the TTL worker to finish before storage teardown, while +also interrupting a long-running client command. The active-pass run uses a single Substrate +worker so a blocking wait on the shutdown coroutine would strand TTL's requests. It uses the test-only +`hangTTLMonitorBeforeStorageAccess` failpoint; it is inactive during normal operation. + ## 5. Inspect the selected MongoDB graph The source list is derived statically from MongoDB's fine-grained `SConscript` library metadata; diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 61f5b24958..a69e000e12 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -936,6 +936,10 @@ void shutdownTask() { serviceContext->setKillAllOperations(); + // TTL is a separate BackgroundJob, not part of the periodic runner. Drain it while + // Data Substrate and its workers are alive, before taking the global shutdown lock. + shutdownTTLBackgroundJob(); + // Shut down the background periodic task runner if (auto runner = serviceContext->getPeriodicRunner()) { runner->shutdown(); diff --git a/src/mongo/db/ttl.cpp b/src/mongo/db/ttl.cpp index 548bfcbdb1..3d390c73ee 100644 --- a/src/mongo/db/ttl.cpp +++ b/src/mongo/db/ttl.cpp @@ -48,6 +48,7 @@ #include "mongo/db/commands/fsync_locked.h" #include "mongo/db/commands/server_status_metric.h" #include "mongo/db/concurrency/write_conflict_exception.h" +#include "mongo/db/coro_sync.h" #include "mongo/db/db_raii.h" #include "mongo/db/exec/delete.h" #include "mongo/db/index/index_descriptor.h" @@ -60,6 +61,7 @@ #include "mongo/util/background.h" #include "mongo/util/concurrency/idle_thread_block.h" #include "mongo/util/exit.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" namespace mongo { @@ -73,6 +75,7 @@ ServerStatusMetricField ttlDeletedDocumentsDisplay("ttl.deletedDocume MONGO_EXPORT_SERVER_PARAMETER(ttlMonitorEnabled, bool, true); MONGO_EXPORT_SERVER_PARAMETER(ttlMonitorSleepSecs, int, 60); // used for testing +MONGO_FAIL_POINT_DEFINE(hangTTLMonitorBeforeStorageAccess); class TTLMonitor : public BackgroundJob { public: @@ -85,15 +88,42 @@ class TTLMonitor : public BackgroundJob { static std::string secondsExpireField; + void shutdown() { + { + stdx::unique_lock lock(_mutex); + _shutdownRequested = true; + _cond.notify_all(); + // Shutdown can run on an Eloq request coroutine. Yield that coroutine while an + // active TTL pass finishes; blocking its worker could strand the pass's requests. + _cond.wait(lock, [this] { return _finished; }); + } + // run() has released its Client and storage resources. BackgroundJob only has its + // completion bookkeeping left, so this native wait cannot block a Substrate request. + wait(); + } + virtual void run() { + ON_BLOCK_EXIT([this] { + stdx::lock_guard lock(_mutex); + _finished = true; + _cond.notify_all(); + }); Client::initThread(name().c_str()); + // Destroy the Client before publishing completion to the shutdown coroutine. ON_BLOCK_EXIT([] { Client::destroy(); }); AuthorizationSession::get(cc())->grantInternalAuthorization(); while (!globalInShutdownDeprecated()) { { + stdx::unique_lock lock(_mutex); MONGO_IDLE_THREAD_BLOCK; - sleepsecs(ttlMonitorSleepSecs.load()); + const auto interval = Seconds(ttlMonitorSleepSecs.load()); + LOG(1) << "TTL monitor waiting for next pass: " << interval.count() << " seconds"; + if (_cond.wait_for(lock, interval.toSystemDuration(), + [this] { return _shutdownRequested; }) || + globalInShutdownDeprecated()) { + break; + } } LOG(3) << "thread awake"; @@ -119,6 +149,11 @@ class TTLMonitor : public BackgroundJob { } private: + coro::Mutex _mutex; + coro::ConditionVariable _cond; + bool _shutdownRequested = false; + bool _finished = false; + void doTTLPass() { const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext(); OperationContext& opCtx = *opCtxPtr; @@ -129,6 +164,18 @@ class TTLMonitor : public BackgroundJob { !repl::ReplicationCoordinator::get(&opCtx)->getMemberState().readable()) return; + // Exercise shutdown with a pass already committed to accessing storage. Releasing + // on the global shutdown flag lets the shutdown command drain it without another + // client connection (listening sockets are already closed during shutdown). + if (MONGO_FAIL_POINT(hangTTLMonitorBeforeStorageAccess)) { + log() << "TTL monitor paused before storage access"; + while (MONGO_FAIL_POINT(hangTTLMonitorBeforeStorageAccess) && + !globalInShutdownDeprecated()) { + sleepmillis(10); + } + log() << "TTL monitor resuming storage access"; + } + // EloqDoc: TTLCollectionCache is unreliable. // // TTLCollectionCache& ttlCollectionCache = @@ -317,15 +364,37 @@ class TTLMonitor : public BackgroundJob { }; namespace { -// The global TTLMonitor object is intentionally leaked. Even though it is only used in one -// function, we declare it here to indicate to the leak sanitizer that the leak of this object -// should not be reported. +// Keep the monitor alive for the process lifetime so repeated shutdown calls can safely wait +// for it. The global pointer also makes this intentional lifetime visible to leak sanitizer. TTLMonitor* ttlMonitor = nullptr; +stdx::mutex ttlMonitorMutex; +bool ttlShutdownRequested = false; } // namespace void startTTLBackgroundJob() { - ttlMonitor = new TTLMonitor(); - ttlMonitor->go(); + stdx::lock_guard lock(ttlMonitorMutex); + if (ttlShutdownRequested) + return; + invariant(!ttlMonitor); + auto monitor = std::make_unique(); + monitor->go(); + ttlMonitor = monitor.release(); +} + +void shutdownTTLBackgroundJob() { + TTLMonitor* monitor; + { + // Shutdown may race startup or run before TTL is initialized. Do not allow a worker + // to be published after shutdown has observed that no worker exists. + stdx::lock_guard lock(ttlMonitorMutex); + ttlShutdownRequested = true; + monitor = ttlMonitor; + } + if (monitor) { + log() << "Stopping TTL monitor"; + monitor->shutdown(); + log() << "TTL monitor stopped"; + } } std::string TTLMonitor::secondsExpireField = "expireAfterSeconds"; diff --git a/src/mongo/db/ttl.h b/src/mongo/db/ttl.h index c015f394ce..f8ec01639c 100644 --- a/src/mongo/db/ttl.h +++ b/src/mongo/db/ttl.h @@ -32,4 +32,6 @@ namespace mongo { void startTTLBackgroundJob(); +// Wake and drain the TTL worker before storage teardown. Safe if startup never started TTL. +void shutdownTTLBackgroundJob(); } From 9f6774688582f148f5a703fb9292928411839de1 Mon Sep 17 00:00:00 2001 From: Liang Jeff Chen Date: Mon, 21 Sep 2026 14:49:06 -0700 Subject: [PATCH 5/5] Enable full CMake CI runtime test coverage Run all retained C++ integration suites, eloq_basic and eloq_core JavaScript suites, and the existing TPCC workload against managed CMake-built server fixtures. Build only the legacy JavaScript test shell with SCons. Require successful tests and clean server shutdown, retain failure diagnostics, isolate Python environments, and serialize shared-server integration tests. Preserve the existing CI matrix and test exclusions. --- .github/scripts/build_cmake_test_shell.sh | 21 +++ .github/scripts/cmake_ci_runtime.sh | 70 ++++++++ .github/workflows/ci.yml | 70 +++++++- cmake/MongoTests.cmake | 6 +- cmake/tests/server_fixture.py | 196 ++++++++++++++++++++++ cmake/tests/server_smoke_config.py | 7 +- cmake/tests/test_server_fixture.py | 125 ++++++++++++++ cmake/tests/test_server_smoke_config.py | 6 + docs/how-to-compile.md | 35 +++- 9 files changed, 529 insertions(+), 7 deletions(-) create mode 100644 .github/scripts/build_cmake_test_shell.sh create mode 100644 .github/scripts/cmake_ci_runtime.sh create mode 100644 cmake/tests/server_fixture.py create mode 100644 cmake/tests/test_server_fixture.py diff --git a/.github/scripts/build_cmake_test_shell.sh b/.github/scripts/build_cmake_test_shell.sh new file mode 100644 index 0000000000..30b0b475d9 --- /dev/null +++ b/.github/scripts/build_cmake_test_shell.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Only the legacy JS test client is built by SCons; never build/install a SCons server here. +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" +shell_build_dir=${1:?usage: $0 shell-build-dir shell-install-prefix} +shell_install_prefix=${2:?usage: $0 shell-build-dir shell-install-prefix} +runtime_env +setup_python2 +cd "$ELOQDOC_BASE_PATH" +run_with_heartbeat "SCons legacy JavaScript test shell" \ + python2 scripts/buildscripts/scons.py --silent \ + MONGO_VERSION=4.0.3 VARIANT_DIR="${SCONS_VARIANT_DIR:-RelWithDebInfo}" \ + CC="${CC:-gcc}" CXX="${CXX:-g++}" \ + CFLAGS="-isystem ${ELOQ_THIRD_PARTY_PREFIX}/include -Wno-nonnull" \ + CXXFLAGS="-isystem ${ELOQ_THIRD_PARTY_PREFIX}/include -Wno-nonnull -Wno-class-memaccess -Wno-interference-size -Wno-redundant-move" \ + CPPDEFINES="ELOQ_MODULE_ENABLED EXT_TX_PROC_ENABLED" \ + LIBPATH="${shell_install_prefix}/lib ${ELOQ_THIRD_PARTY_PREFIX}/lib ${ELOQ_THIRD_PARTY_PREFIX}/lib64" \ + --build-dir="$shell_build_dir" --prefix="$shell_install_prefix" \ + --allocator=system --link-model=dynamic --install-mode=hygienic \ + --disable-warnings-as-errors --jlink=1 -j"$BUILD_JOBS" install-shell +"${shell_install_prefix}/bin/eloqdoc-cli" --version diff --git a/.github/scripts/cmake_ci_runtime.sh b/.github/scripts/cmake_ci_runtime.sh new file mode 100644 index 0000000000..d78e6547c7 --- /dev/null +++ b/.github/scripts/cmake_ci_runtime.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# One disposable server and S3 fixture per test phase. No build work runs here. +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" +build_dir=$(realpath "${1:?usage: $0 cmake-build-dir integration|jstests|tpcc}") +phase=${2:?usage: $0 cmake-build-dir integration|jstests|tpcc} +cd "$ELOQDOC_BASE_PATH" +runtime_env + +# Read the actual configured backend; never silently test a different storage engine. +cache_value() { sed -n "s/^$1:[^=]*=//p" "$build_dir/CMakeCache.txt"; } +data_store=$(cache_value WITH_DATA_STORE) +log_state=$(cache_value WITH_LOG_STATE) +port=27017 +case "$phase" in + integration) + if [ "$(cache_value ELOQDOC_REGISTER_INTEGRATION_TESTS)" != ON ]; then + echo "Configure with -DELOQDOC_REGISTER_INTEGRATION_TESTS=ON first" >&2 + exit 1 + fi + connection_string=$(cache_value ELOQDOC_TEST_CONNECTION_STRING) + if [[ ! "$connection_string" =~ ^(localhost|127\.0\.0\.1):([0-9]+)$ ]]; then + echo "The managed integration fixture requires a localhost:port connection string" >&2 + exit 1 + fi + port=${BASH_REMATCH[2]} + # These suites share failpoints and global server parameters; keep them serial. + command=(ctest --test-dir "$build_dir" -L '^integration$' --no-tests=error + --output-on-failure -j1) + phase_timeout=${INTEGRATION_TIMEOUT_SECONDS:-4800} + ;; + jstests) + setup_python2 + test_shell=${ELOQDOC_TEST_SHELL:?Set ELOQDOC_TEST_SHELL to eloqdoc-cli} + test_shell=$(realpath "$test_shell") + export PATH="$(dirname "$test_shell"):$PATH" + command=(python2 scripts/buildscripts/resmoke.py --mongo="$test_shell" + --suites=eloq_basic,eloq_core '--shellPort={port}' --jobs=1 + --continueOnFailure '--dbpathPrefix={fixture}/resmoke-data' + "--reportFile=$build_dir/jstests-results.json") + phase_timeout=${JSTEST_TIMEOUT_SECONDS:-7200} + ;; + tpcc) + # Reuse exactly the SCons CI workload (two warehouses, transactional driver). + command=(bash -c 'source "$1/.github/scripts/common.sh"; run_tpcc "$2"' + bash "$ELOQDOC_BASE_PATH" "$build_dir/tpcc-client") + phase_timeout=${TPCC_PHASE_TIMEOUT_SECONDS:-2400} + ;; + *) echo "Unknown runtime phase: $phase" >&2; exit 1 ;; +esac + +finish_runtime() { + local status=$? + if [ "$status" -ne 0 ] && [ -n "${RUSTFS_RUN_DIR:-}" ] && [ -f /tmp/rustfs.log ]; then + mkdir -p "$build_dir/runtime-diagnostics" + cp /tmp/rustfs.log "$build_dir/runtime-diagnostics/rustfs-${phase}.log" + fi + stop_rustfs + exit "$status" +} +trap finish_runtime EXIT +if needs_s3 "$data_store" "$log_state"; then + start_rustfs "$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" +fi +"${CMAKE_TEST_PYTHON:-/usr/bin/python3}" cmake/tests/server_fixture.py \ + --server "$build_dir/eloqdoc" --data-store "$data_store" --log-state "$log_state" \ + --label "$phase" --timeout "$phase_timeout" \ + --port "$port" \ + --memory-limit-mb "${NODE_MEMORY_LIMIT_MB:-4000}" \ + --diagnostics-dir "$build_dir/runtime-diagnostics" -- "${command[@]}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe673e35e1..f1d9d82784 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,19 @@ jobs: apt-get install -y --no-install-recommends python3-yaml /usr/bin/python3 -c 'import yaml' + - name: Prepare runtime test clients + id: cmake_clients + working-directory: eloqdoc + shell: bash + run: | + apt-get install -y --no-install-recommends python3-venv + /usr/bin/python3 -m pip install --target build/cmake-ci/python 'pymongo==4.8.0' + source .github/scripts/common.sh + setup_python2 + python2 -c 'import yaml, pymongo, requests' + git clone --depth 1 --branch production \ + https://github.com/eloqdata/py-tpcc.git "$PY_TPCC_PATH" + - name: Configure top-level CMake build working-directory: eloqdoc run: | @@ -107,6 +120,8 @@ jobs: -DCMAKE_JOB_POOL_LINK=link_pool \ -DBUILD_SHARED_LIBS=ON \ -DELOQDOC_BUILD_TESTS=ON \ + -DELOQDOC_REGISTER_INTEGRATION_TESTS=ON \ + -DELOQDOC_TEST_CONNECTION_STRING=localhost:27017 \ -DELOQ_THIRD_PARTY_PREFIX="$ELOQ_THIRD_PARTY_PREFIX" \ -DELOQ_THIRD_PARTY_REQUIRED=ON \ -DWITH_DATA_STORE=ELOQDSS_ELOQSTORE \ @@ -122,11 +137,61 @@ jobs: cmake --build build/cmake-ci \ --target eloqdoc eloqdoc-tests - - name: Run all registered CMake tests + - name: Run CMake unit and integration-startup tests working-directory: eloqdoc run: | ctest --test-dir build/cmake-ci \ - --output-on-failure + --label-exclude '^integration$' --no-tests=error --output-on-failure + + - name: Run C++ integration suites against the CMake server + id: cmake_integration + if: ${{ !cancelled() && steps.cmake_build.outcome == 'success' && steps.cmake_clients.outcome == 'success' }} + working-directory: eloqdoc + shell: bash + run: | + PYTHONPATH="$PWD/build/cmake-ci/python" \ + bash .github/scripts/cmake_ci_runtime.sh build/cmake-ci integration + + - name: Build the legacy JavaScript test shell only + id: cmake_shell + if: ${{ !cancelled() && steps.cmake_build.outcome == 'success' && steps.cmake_clients.outcome == 'success' }} + working-directory: eloqdoc + shell: bash + run: | + BUILD_JOBS="$CMAKE_BUILD_PARALLEL_LEVEL" \ + bash .github/scripts/build_cmake_test_shell.sh \ + "$PWD/build/cmake-test-shell" "$PWD/build/cmake-test-client" + + - name: Run eloq_basic and eloq_core against the CMake server + id: cmake_jstests + if: ${{ !cancelled() && steps.cmake_shell.outcome == 'success' }} + working-directory: eloqdoc + shell: bash + run: | + ELOQDOC_TEST_SHELL="$PWD/build/cmake-test-client/bin/eloqdoc-cli" \ + PYTHONPATH="$PWD/build/cmake-ci/python" \ + bash .github/scripts/cmake_ci_runtime.sh build/cmake-ci jstests + + - name: Run TPCC against the CMake server + id: cmake_tpcc + if: ${{ !cancelled() && steps.cmake_build.outcome == 'success' && steps.cmake_clients.outcome == 'success' }} + working-directory: eloqdoc + shell: bash + run: | + PYTHONPATH="$PWD/build/cmake-ci/python" \ + bash .github/scripts/cmake_ci_runtime.sh build/cmake-ci tpcc + + - name: Preserve failed CMake runtime-test logs + if: ${{ failure() && (steps.cmake_integration.outcome == 'failure' || steps.cmake_jstests.outcome == 'failure' || steps.cmake_tpcc.outcome == 'failure') }} + uses: actions/upload-artifact@v4 + with: + name: cmake-runtime-eloqstore-s3-amd64-${{ github.run_attempt }} + path: | + eloqdoc/build/cmake-ci/runtime-diagnostics/ + eloqdoc/build/cmake-ci/jstests-results.json + eloqdoc/build/cmake-ci/Testing/Temporary/ + if-no-files-found: warn + retention-days: 7 - name: Exercise the normal server over the MongoDB wire protocol id: cmake_smoke @@ -137,7 +202,6 @@ jobs: source .github/scripts/common.sh trap 'stop_rustfs' EXIT start_rustfs "$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" - /usr/bin/python3 -m pip install --target build/cmake-ci/python 'pymongo==4.8.0' PYTHONPATH="$PWD/build/cmake-ci/python" \ cmake --build build/cmake-ci --target eloqdoc-server-smoke diff --git a/cmake/MongoTests.cmake b/cmake/MongoTests.cmake index 33e516e506..942e8bcd1a 100644 --- a/cmake/MongoTests.cmake +++ b/cmake/MongoTests.cmake @@ -19,6 +19,9 @@ set_tests_properties(cmake/server-smoke-config PROPERTIES LABELS "cmake;unit" TI add_test(NAME cmake/server-smoke-diagnostics COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_server_smoke_diagnostics.py") set_tests_properties(cmake/server-smoke-diagnostics PROPERTIES LABELS "cmake;unit" TIMEOUT 60) +add_test(NAME cmake/server-fixture + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_server_fixture.py") +set_tests_properties(cmake/server-fixture PROPERTIES LABELS "cmake;unit" TIMEOUT 30) # Opt-in wire-protocol regression fixture; PyMongo is an explicit test-only dependency. add_custom_target(eloqdoc-server-smoke @@ -180,7 +183,8 @@ foreach(_test_id IN LISTS ELOQDOC_MONGO_TEST_IDS) add_test(NAME "${_name}" COMMAND ${_target} "--connectionString=${ELOQDOC_TEST_CONNECTION_STRING}") set_tests_properties("${_name}" PROPERTIES - LABELS "mongo;integration" TIMEOUT 600) + LABELS "mongo;integration" TIMEOUT 600 + RESOURCE_LOCK eloqdoc-integration-server) endif() endif() diff --git a/cmake/tests/server_fixture.py b/cmake/tests/server_fixture.py new file mode 100644 index 0000000000..f582d07a51 --- /dev/null +++ b/cmake/tests/server_fixture.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Run a command against a private EloqDoc server; require clean server shutdown. + +The command may use {port}, {connection_string}, and {fixture} placeholders. +Unlike resmoke's external-server mode, this wrapper owns and checks the server. +Requires PyMongo in this interpreter, not in the child command's environment. +""" + +import argparse +import json +import os +from pathlib import Path +import signal +import socket +import subprocess +import tempfile +import time + +from server_smoke_config import substrate_config +from server_smoke_diagnostics import describe_exit, report_failure, wait_for_clean_shutdown + + +def reserve_ports(port): + # Substrate also uses ports adjacent to tx_port. Hold all reservations until + # immediately before launch, and fail rather than connect to someone else's server. + sockets = [] + try: + listener = socket.socket() + sockets.append(listener) + # Match the server listener: permit reuse after the preceding fixture's + # connections enter TIME_WAIT, but do not use SO_REUSEPORT/shared listeners. + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", port)) + port = listener.getsockname()[1] + for _ in range(100): + group = [] + try: + for offset in range(10): + sock = socket.socket() + group.append(sock) + sock.bind(("127.0.0.1", 0 if offset == 0 else tx_port + offset)) + if offset == 0: + tx_port = sock.getsockname()[1] + if tx_port > 65525: + raise OSError("not enough adjacent ports") + sockets.extend(group) + break + except OSError: + for sock in group: + sock.close() + else: + raise RuntimeError("Could not reserve Data Substrate ports") + host_manager = socket.socket() + sockets.append(host_manager) + host_manager.bind(("127.0.0.1", 0)) + return sockets, port, tx_port, host_manager.getsockname()[1] + except BaseException: + for sock in sockets: + sock.close() + raise + + +def child_environment(environment): + result = dict(environment) + # The wrapper uses Python 3 PyMongo. Never inject it into Python 2 resmoke + # or TPCC's independently provisioned Python 3 virtual environment. + result.pop("PYTHONPATH", None) + return result + + +def expand_command(command, port, root): + replacements = {"{port}": str(port), "{connection_string}": f"127.0.0.1:{port}", + "{fixture}": str(root)} + for old, new in replacements.items(): + command = [argument.replace(old, new) for argument in command] + return command + + +def stop_process_group(process): + # Only kill groups we created. Even when their leader exited, test clients + # and the forked host manager must not leak into the next CI phase. + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + return + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + + +def run_command(command, root, timeout): + # Retain complete output and print it live for CI's heartbeat and test summaries. + # Using a file avoids a blocked pipe if a test or one of its descendants hangs. + with (root / "tests.log").open("wb") as output: + process = subprocess.Popen(command, stdout=output, stderr=subprocess.STDOUT, + env=child_environment(os.environ), start_new_session=True) + try: + started = time.monotonic() + with (root / "tests.log").open(errors="replace") as stream: + while process.poll() is None: + print(stream.read(), end="", flush=True) + if time.monotonic() - started > timeout: + raise TimeoutError(f"Test command timed out after {timeout}s") + time.sleep(0.5) + print(stream.read(), end="", flush=True) + if process.returncode: + raise RuntimeError("Test command failed: " + describe_exit(process.returncode)) + finally: + stop_process_group(process) + + +def run_fixture(args): + from pymongo import MongoClient + from pymongo.errors import AutoReconnect, PyMongoError + + root = Path(tempfile.mkdtemp(prefix=f"eloqdoc-{args.label}-")) + print(f"Fixture: {root}", flush=True) + (root / "db").mkdir() + reservations, port, tx_port, hm_port = reserve_ports(args.port) + try: + (root / "substrate.cnf").write_text(substrate_config( + root, tx_port, hm_port, args.data_store, args.log_state, os.environ, + memory_limit_mb=args.memory_limit_mb), encoding="utf-8") + (root / "server.json").write_text(json.dumps({ + "storage": {"engine": "eloq", "dbPath": str(root / "db")}, + "net": {"port": port, "bindIp": "127.0.0.1", "serviceExecutor": "adaptive", + "adaptiveThreadNum": 1}, + "setParameter": {"enableTestCommands": True, "diagnosticDataCollectionEnabled": False, + "disableLogicalSessionCacheRefresh": True}}), encoding="utf-8") + finally: + for sock in reservations: + sock.close() + with (root / "server.log").open("wb") as log: + server = subprocess.Popen([ + str(args.server.resolve()), "--config=" + str(root / "server.json"), + "--data_substrate_config=" + str(root / "substrate.cnf"), "--log_dir=" + str(root)], + cwd=root, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + client = MongoClient("127.0.0.1", port, directConnection=True, + serverSelectionTimeoutMS=1000, socketTimeoutMS=10000) + try: + deadline = time.monotonic() + 90 + while True: + if server.poll() is not None: + raise RuntimeError("Server exited before readiness: " + describe_exit(server.returncode)) + try: + client.admin.command("ping") + break + except PyMongoError: + if time.monotonic() >= deadline: + raise TimeoutError("Server readiness timed out") + time.sleep(0.5) + run_command(expand_command(args.command, port, root), root, args.timeout) + # A successful client/test command must not hide a server crash. + if server.poll() is not None: + raise RuntimeError("Server exited during tests: " + describe_exit(server.returncode)) + try: + client.admin.command("shutdown", force=True) + except AutoReconnect: + pass # Disconnect is expected; the process exit code is authoritative. + wait_for_clean_shutdown(server, timeout=60) + print(f"PASS {args.label}: test command and server shutdown", flush=True) + except BaseException as exc: + report_failure(root, server.poll(), exc, args.diagnostics_dir) + raise + finally: + client.close() + stop_process_group(server) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--server", required=True, type=Path) + parser.add_argument("--data-store", required=True) + parser.add_argument("--log-state", required=True) + parser.add_argument("--label", choices=("integration", "jstests", "tpcc"), required=True) + parser.add_argument("--port", type=int, default=27017) + parser.add_argument("--memory-limit-mb", type=int, default=4000) + parser.add_argument("--timeout", type=int, default=7200) + parser.add_argument("--diagnostics-dir", type=Path) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + if args.command[:1] == ["--"]: + args.command.pop(0) + if not args.command: + parser.error("a test command after -- is required") + run_fixture(args) + + +if __name__ == "__main__": + main() diff --git a/cmake/tests/server_smoke_config.py b/cmake/tests/server_smoke_config.py index d060e139a9..49b63cffa6 100644 --- a/cmake/tests/server_smoke_config.py +++ b/cmake/tests/server_smoke_config.py @@ -3,11 +3,14 @@ import uuid -def substrate_config(root, tx_port, hm_port, data_store, log_state, environment, *, core_number=2): +def substrate_config(root, tx_port, hm_port, data_store, log_state, environment, *, core_number=2, + memory_limit_mb=None): cloud = (data_store, log_state) == ("ELOQDSS_ELOQSTORE", "ROCKSDB_CLOUD_S3") local = data_store in ("ROCKSDB", "ELOQDSS_ROCKSDB") and log_state == "ROCKSDB" if not (cloud or local): raise ValueError(f"Unsupported smoke-test backend: {data_store}/{log_state}") + if memory_limit_mb is None: + memory_limit_mb = 2048 if cloud else 512 cloud_local, cloud_store = "", "" if cloud: @@ -42,7 +45,7 @@ def substrate_config(root, tx_port, hm_port, data_store, log_state, environment, return ( f"[local]\ncore_number={core_number}\n" - f"node_memory_limit_mb={2048 if cloud else 512}\n" + f"node_memory_limit_mb={memory_limit_mb}\n" f"enable_data_store=true\nenable_wal={'true' if cloud else 'false'}\n" "enable_mvcc=true\nevent_dispatcher_num=1\nbind_all=false\n" f"tx_ip=127.0.0.1\ntx_port={tx_port}\nhm_port={hm_port}\n" diff --git a/cmake/tests/test_server_fixture.py b/cmake/tests/test_server_fixture.py new file mode 100644 index 0000000000..6f3b32ef3d --- /dev/null +++ b/cmake/tests/test_server_fixture.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Regression checks for CI scope, client isolation, and runtime result propagation.""" + +import contextlib +import io +from pathlib import Path +import socket +import sys +import tempfile +import types +import unittest +from unittest import mock + +import yaml + +from server_fixture import child_environment, expand_command, reserve_ports, run_command, run_fixture + + +class ServerFixtureTest(unittest.TestCase): + def test_port_reservation_allows_time_wait_but_rejects_a_busy_port(self): + listener = mock.Mock() + listener.bind.side_effect = OSError("Address already in use") + with mock.patch("server_fixture.socket.socket", return_value=listener): + with self.assertRaisesRegex(OSError, "Address already in use"): + reserve_ports(27017) + listener.setsockopt.assert_called_once_with(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.close.assert_called_once() + + def test_client_python_environments_are_isolated(self): + environment = {"PYTHONPATH": "/python3/pymongo", "PATH": "/shell/bin:/usr/bin", + "LD_LIBRARY_PATH": "/third-party/lib"} + self.assertEqual({key: value for key, value in environment.items() if key != "PYTHONPATH"}, + child_environment(environment)) + self.assertIn("PYTHONPATH", environment) + + def test_command_arguments_are_substituted_without_shell_expansion(self): + command = ["ctest", "--connection={connection_string}", "--port={port}", + "--report={fixture}/report.json", "{not_a_placeholder}"] + self.assertEqual(["ctest", "--connection=127.0.0.1:12345", "--port=12345", + "--report=/tmp/path with spaces/report.json", "{not_a_placeholder}"], + expand_command(command, 12345, Path("/tmp/path with spaces"))) + + def test_test_failure_is_not_hidden_by_fixture_cleanup(self): + with tempfile.TemporaryDirectory() as directory, contextlib.redirect_stdout(io.StringIO()): + root = Path(directory) + with self.assertRaisesRegex(RuntimeError, "exit code 23"): + run_command([sys.executable, "-c", "print('failure context'); raise SystemExit(23)"], + root, 5) + self.assertIn("failure context", (root / "tests.log").read_text()) + + def test_hung_command_times_out(self): + with tempfile.TemporaryDirectory() as directory, contextlib.redirect_stdout(io.StringIO()): + with self.assertRaisesRegex(TimeoutError, "timed out"): + run_command([sys.executable, "-c", "import time; time.sleep(60)"], Path(directory), 1) + + def test_successful_command_is_recorded(self): + with tempfile.TemporaryDirectory() as directory, contextlib.redirect_stdout(io.StringIO()): + root = Path(directory) + run_command([sys.executable, "-c", "print('passed')"], root, 5) + self.assertEqual("passed\n", (root / "tests.log").read_text()) + + def check_server_exit(self, returncode): + # No PyMongo installation or network access needed for these lifecycle checks. + client = mock.Mock() + pymongo = types.ModuleType("pymongo") + pymongo.MongoClient = mock.Mock(return_value=client) + errors = types.ModuleType("pymongo.errors") + errors.PyMongoError = type("PyMongoError", (Exception,), {}) + errors.AutoReconnect = type("AutoReconnect", (errors.PyMongoError,), {}) + server = mock.Mock() + server.poll.side_effect = [None, None, returncode] + server.wait.return_value = returncode + with tempfile.TemporaryDirectory() as directory, contextlib.ExitStack() as stack: + stack.enter_context(contextlib.redirect_stdout(io.StringIO())) + stack.enter_context(mock.patch.dict(sys.modules, {"pymongo": pymongo, + "pymongo.errors": errors})) + stack.enter_context(mock.patch("server_fixture.tempfile.mkdtemp", return_value=directory)) + stack.enter_context(mock.patch("server_fixture.reserve_ports", + return_value=([], 27017, 16379, 16389))) + stack.enter_context(mock.patch("server_fixture.subprocess.Popen", return_value=server)) + stack.enter_context(mock.patch("server_fixture.run_command")) + cleanup = stack.enter_context(mock.patch("server_fixture.stop_process_group")) + report = stack.enter_context(mock.patch("server_fixture.report_failure")) + args = types.SimpleNamespace(label="integration", port=27017, memory_limit_mb=4000, + server=Path("/tmp/eloqdoc"), data_store="ELOQDSS_ROCKSDB", + log_state="ROCKSDB", command=["ctest"], timeout=10, + diagnostics_dir=None) + if returncode: + with self.assertRaisesRegex(AssertionError, "server shutdown failed"): + run_fixture(args) + report.assert_called_once() + else: + run_fixture(args) + report.assert_not_called() + cleanup.assert_called_once_with(server) + client.close.assert_called_once() + client.admin.command.assert_any_call("shutdown", force=True) + + def test_passing_tests_do_not_hide_a_server_crash(self): + self.check_server_exit(-11) + + def test_passing_tests_require_clean_shutdown(self): + self.check_server_exit(0) + + def test_ci_enables_all_runtime_phases(self): + source = Path(__file__).resolve().parents[2] + workflow = yaml.safe_load((source / ".github/workflows/ci.yml").read_text()) + job = workflow["jobs"]["cmake-build-smoke"] + self.assertNotIn("strategy", job) + commands = "\n".join(step.get("run", "") for step in job["steps"]) + self.assertIn("-DELOQDOC_REGISTER_INTEGRATION_TESTS=ON", commands) + for phase in ("integration", "jstests", "tpcc"): + self.assertIn("cmake_ci_runtime.sh build/cmake-ci " + phase, commands) + runtime = (source / ".github/scripts/cmake_ci_runtime.sh").read_text() + self.assertIn("--suites=eloq_basic,eloq_core", runtime) + self.assertIn("--no-tests=error", runtime) + self.assertIn('--server "$build_dir/eloqdoc"', runtime) + shell = (source / ".github/scripts/build_cmake_test_shell.sh").read_text() + self.assertIn("install-shell", shell) + self.assertNotIn("install-core", shell) + self.assertNotIn("install-servers", shell) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/tests/test_server_smoke_config.py b/cmake/tests/test_server_smoke_config.py index b0b017fb92..da94ea3ddd 100644 --- a/cmake/tests/test_server_smoke_config.py +++ b/cmake/tests/test_server_smoke_config.py @@ -82,6 +82,12 @@ def test_active_shutdown_can_use_a_single_worker(self): self.assertEqual(1, self.config(data_store, log_state, core_number=1)[ "local"].getint("core_number")) + def test_larger_runtime_suites_can_set_the_memory_budget(self): + config = configparser.ConfigParser() + config.read_string(substrate_config(Path("/tmp/runtime"), 12345, 12346, + "ELOQDSS_ROCKSDB", "ROCKSDB", {}, memory_limit_mb=4000)) + self.assertEqual(4000, config["local"].getint("node_memory_limit_mb")) + if __name__ == "__main__": unittest.main() diff --git a/docs/how-to-compile.md b/docs/how-to-compile.md index f7a76a0961..27d53e9aa8 100644 --- a/docs/how-to-compile.md +++ b/docs/how-to-compile.md @@ -244,7 +244,7 @@ run their tests in their own repositories. MongoDB integration tests require an already-running EloqDoc fixture, so their binaries are built but their server-dependent cases are not registered with CTest by default. Every integration binary has an always-registered `startup-options` check which runs `--help` without a server; -this catches missing option-parser initializers even in normal CI. To register the full cases, +this catches missing option-parser initializers without a server. To register the full cases, configure with: ```bash @@ -255,6 +255,39 @@ configure with: Then run `ctest --test-dir build/cmake --label-regex integration --output-on-failure` after starting the fixture. +GitHub's CMake job explicitly enables `ELOQDOC_REGISTER_INTEGRATION_TESTS`. It runs unit and +integration-startup checks first, then runs the actual C++ integration suites against a managed +CMake-built server. These suites execute serially because they modify shared server parameters +and failpoints. The same job runs the existing `eloq_basic` and `eloq_core` JavaScript suites, +the same two-warehouse transactional TPCC workload as SCons CI, and the wire-protocol smoke tests. +The CI matrix remains one amd64 RelWithDebInfo EloqStore/S3 configuration for CMake. + +The legacy JavaScript client is built separately with SCons's `install-shell` target; this does +not build or substitute a SCons server. Python 2 is needed for that shell build and resmoke, +while the server fixture uses Python 3/PyMongo. Their Python package paths are isolated. +Each runtime phase gets fresh local data and a private S3 bucket on its disposable RustFS +service, and must pass both its tests and clean server shutdown. Failure logs are preserved in +`build/cmake-ci/runtime-diagnostics/`; configs, credentials, and data directories are not uploaded. + +To reproduce individual CI runtime phases locally (cloud configurations also need RustFS on +`PATH` and the S3 environment variables documented below): + +```bash +cmake -S . -B build/cmake -DELOQDOC_REGISTER_INTEGRATION_TESTS=ON \ + -DELOQDOC_TEST_CONNECTION_STRING=localhost:27017 +cmake --build build/cmake --target eloqdoc eloqdoc-tests -j8 +bash .github/scripts/cmake_ci_runtime.sh build/cmake integration +ELOQDOC_TEST_SHELL=/path/to/eloqdoc-cli \ + bash .github/scripts/cmake_ci_runtime.sh build/cmake jstests +PY_TPCC_PATH=/path/to/py-tpcc \ + bash .github/scripts/cmake_ci_runtime.sh build/cmake tpcc +``` + +The fixture interpreter defaults to `/usr/bin/python3` and can be selected with +`CMAKE_TEST_PYTHON`; install `pymongo==4.8.0` for that interpreter. TPCC uses its own virtual +environment. The JavaScript and TPCC phases reserve port 27017 and refuse to reuse an existing +server on that port. No Data Substrate/data-store test suites are added. + The existing `eloq_basic` and `eloq_core` JavaScript suites can also run against the CMake server using an EloqDoc shell. Put the shell directory on `PATH` as well as passing resmoke's `--mongo` option, because some tests spawn `eloqdoc-cli` by name. `set_param1.js` always runs