diff --git a/.github/scripts/build_cmake_test_shell.sh b/.github/scripts/build_cmake_test_shell.sh new file mode 100644 index 0000000000..30b0b475d9 --- /dev/null +++ b/.github/scripts/build_cmake_test_shell.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Only the legacy JS test client is built by SCons; never build/install a SCons server here. +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" +shell_build_dir=${1:?usage: $0 shell-build-dir shell-install-prefix} +shell_install_prefix=${2:?usage: $0 shell-build-dir shell-install-prefix} +runtime_env +setup_python2 +cd "$ELOQDOC_BASE_PATH" +run_with_heartbeat "SCons legacy JavaScript test shell" \ + python2 scripts/buildscripts/scons.py --silent \ + MONGO_VERSION=4.0.3 VARIANT_DIR="${SCONS_VARIANT_DIR:-RelWithDebInfo}" \ + CC="${CC:-gcc}" CXX="${CXX:-g++}" \ + CFLAGS="-isystem ${ELOQ_THIRD_PARTY_PREFIX}/include -Wno-nonnull" \ + CXXFLAGS="-isystem ${ELOQ_THIRD_PARTY_PREFIX}/include -Wno-nonnull -Wno-class-memaccess -Wno-interference-size -Wno-redundant-move" \ + CPPDEFINES="ELOQ_MODULE_ENABLED EXT_TX_PROC_ENABLED" \ + LIBPATH="${shell_install_prefix}/lib ${ELOQ_THIRD_PARTY_PREFIX}/lib ${ELOQ_THIRD_PARTY_PREFIX}/lib64" \ + --build-dir="$shell_build_dir" --prefix="$shell_install_prefix" \ + --allocator=system --link-model=dynamic --install-mode=hygienic \ + --disable-warnings-as-errors --jlink=1 -j"$BUILD_JOBS" install-shell +"${shell_install_prefix}/bin/eloqdoc-cli" --version diff --git a/.github/scripts/cmake_ci_runtime.sh b/.github/scripts/cmake_ci_runtime.sh new file mode 100644 index 0000000000..d78e6547c7 --- /dev/null +++ b/.github/scripts/cmake_ci_runtime.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# One disposable server and S3 fixture per test phase. No build work runs here. +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/common.sh" +build_dir=$(realpath "${1:?usage: $0 cmake-build-dir integration|jstests|tpcc}") +phase=${2:?usage: $0 cmake-build-dir integration|jstests|tpcc} +cd "$ELOQDOC_BASE_PATH" +runtime_env + +# Read the actual configured backend; never silently test a different storage engine. +cache_value() { sed -n "s/^$1:[^=]*=//p" "$build_dir/CMakeCache.txt"; } +data_store=$(cache_value WITH_DATA_STORE) +log_state=$(cache_value WITH_LOG_STATE) +port=27017 +case "$phase" in + integration) + if [ "$(cache_value ELOQDOC_REGISTER_INTEGRATION_TESTS)" != ON ]; then + echo "Configure with -DELOQDOC_REGISTER_INTEGRATION_TESTS=ON first" >&2 + exit 1 + fi + connection_string=$(cache_value ELOQDOC_TEST_CONNECTION_STRING) + if [[ ! "$connection_string" =~ ^(localhost|127\.0\.0\.1):([0-9]+)$ ]]; then + echo "The managed integration fixture requires a localhost:port connection string" >&2 + exit 1 + fi + port=${BASH_REMATCH[2]} + # These suites share failpoints and global server parameters; keep them serial. + command=(ctest --test-dir "$build_dir" -L '^integration$' --no-tests=error + --output-on-failure -j1) + phase_timeout=${INTEGRATION_TIMEOUT_SECONDS:-4800} + ;; + jstests) + setup_python2 + test_shell=${ELOQDOC_TEST_SHELL:?Set ELOQDOC_TEST_SHELL to eloqdoc-cli} + test_shell=$(realpath "$test_shell") + export PATH="$(dirname "$test_shell"):$PATH" + command=(python2 scripts/buildscripts/resmoke.py --mongo="$test_shell" + --suites=eloq_basic,eloq_core '--shellPort={port}' --jobs=1 + --continueOnFailure '--dbpathPrefix={fixture}/resmoke-data' + "--reportFile=$build_dir/jstests-results.json") + phase_timeout=${JSTEST_TIMEOUT_SECONDS:-7200} + ;; + tpcc) + # Reuse exactly the SCons CI workload (two warehouses, transactional driver). + command=(bash -c 'source "$1/.github/scripts/common.sh"; run_tpcc "$2"' + bash "$ELOQDOC_BASE_PATH" "$build_dir/tpcc-client") + phase_timeout=${TPCC_PHASE_TIMEOUT_SECONDS:-2400} + ;; + *) echo "Unknown runtime phase: $phase" >&2; exit 1 ;; +esac + +finish_runtime() { + local status=$? + if [ "$status" -ne 0 ] && [ -n "${RUSTFS_RUN_DIR:-}" ] && [ -f /tmp/rustfs.log ]; then + mkdir -p "$build_dir/runtime-diagnostics" + cp /tmp/rustfs.log "$build_dir/runtime-diagnostics/rustfs-${phase}.log" + fi + stop_rustfs + exit "$status" +} +trap finish_runtime EXIT +if needs_s3 "$data_store" "$log_state"; then + start_rustfs "$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" +fi +"${CMAKE_TEST_PYTHON:-/usr/bin/python3}" cmake/tests/server_fixture.py \ + --server "$build_dir/eloqdoc" --data-store "$data_store" --log-state "$log_state" \ + --label "$phase" --timeout "$phase_timeout" \ + --port "$port" \ + --memory-limit-mb "${NODE_MEMORY_LIMIT_MB:-4000}" \ + --diagnostics-dir "$build_dir/runtime-diagnostics" -- "${command[@]}" diff --git a/.github/scripts/common.sh b/.github/scripts/common.sh index b21e72453b..3735b0a6d8 100755 --- a/.github/scripts/common.sh +++ b/.github/scripts/common.sh @@ -28,6 +28,7 @@ fi export ELOQ_THIRD_PARTY_REQUIRED="${ELOQ_THIRD_PARTY_REQUIRED:-ON}" export BUILD_JOBS="${BUILD_JOBS:-$(nproc)}" [ "${BUILD_JOBS}" -lt 1 ] && BUILD_JOBS=1 +[ "${BUILD_JOBS}" -gt 8 ] && BUILD_JOBS=8 export CMAKE_BUILD_TIMEOUT_SECONDS="${CMAKE_BUILD_TIMEOUT_SECONDS:-3600}" export SCONS_BUILD_TIMEOUT_SECONDS="${SCONS_BUILD_TIMEOUT_SECONDS:-7200}" export TPCC_TIMEOUT_SECONDS="${TPCC_TIMEOUT_SECONDS:-1200}" @@ -278,7 +279,9 @@ build_eloqdoc() { aarch64|arm64) scons_arch_flags="-march=armv8-a+crc" ;; esac - local scons_third_party_include="-idirafter ${ELOQ_THIRD_PARTY_PREFIX}/include" + # Keep MongoDB's explicit -I paths first, but prefer the selected dependency + # prefix over unrelated copies installed in the compiler's default paths. + local scons_third_party_include="-isystem ${ELOQ_THIRD_PARTY_PREFIX}/include" local scons_cflags="${scons_third_party_include} ${scons_arch_flags} -Wno-nonnull" local scons_cxxflags="${scons_third_party_include} ${scons_arch_flags} -Wno-nonnull -Wno-class-memaccess -Wno-interference-size -Wno-redundant-move" local scons_libpath="${ELOQ_THIRD_PARTY_PREFIX}/lib ${ELOQ_THIRD_PARTY_PREFIX}/lib64" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 87cf0a2fce..41b31d216a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -99,6 +99,9 @@ jobs: VARIANT_ID: ${{ matrix.id }} DATA_STORE_TYPE: ${{ matrix.data_store }} WITH_LOG_STATE: ${{ matrix.log_state }} + # This release job uses the historical SCons packaging path. Pin Python 2 explicitly even + # though the compatibility installer currently enables it by default. + ELOQ_INSTALL_LEGACY_SCONS: 1 steps: - name: Bootstrap checkout tools diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3327223535..f1d9d82784 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,12 +17,8 @@ concurrency: jobs: prime-third-party: - name: prime-third-party (${{ matrix.arch }}) - strategy: - fail-fast: false - matrix: - arch: [amd64, arm64] - runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + name: prime-third-party (amd64) + runs-on: ubuntu-24.04 container: image: eloqdata/ubuntu-dev:24.04 options: --user root @@ -46,9 +42,180 @@ jobs: uses: ./eloqdoc/.github/actions/setup-third-party with: source-dir: ${{ github.workspace }}/eloqdoc - arch: ${{ matrix.arch }} + arch: amd64 + cache-scope: ubuntu-dev + + cmake-build-smoke: + name: CMake/Ninja RelWithDebInfo-eloqstore_s3-amd64 + needs: prime-third-party + runs-on: ubuntu-24.04 + timeout-minutes: 240 + container: + image: eloqdata/ubuntu-dev:24.04 + options: --user root --privileged --security-opt seccomp=unconfined --ulimit memlock=-1:-1 + env: + ELOQ_THIRD_PARTY_PREFIX: /opt/eloq/third_party + CMAKE_PREFIX_PATH: /opt/eloq/third_party + CMAKE_INCLUDE_PATH: /opt/eloq/third_party/include + CMAKE_LIBRARY_PATH: /opt/eloq/third_party/lib + LD_LIBRARY_PATH: /opt/eloq/third_party/lib:/opt/eloq/third_party/lib64 + S3_ENDPOINT: http://127.0.0.1:9900 + S3_ACCESS_KEY: rustfsadmin + S3_SECRET_KEY: rustfsadmin + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: false + path: eloqdoc + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Checkout product submodules + run: | + git config --global --add safe.directory '*' + cd "$GITHUB_WORKSPACE/eloqdoc" + bash scripts/checkout_product_submodules.sh + + - name: Setup third-party (cached) + uses: ./eloqdoc/.github/actions/setup-third-party + with: + source-dir: ${{ github.workspace }}/eloqdoc + arch: amd64 cache-scope: ubuntu-dev + - name: Limit CMake build and test parallelism to runner capacity + run: | + jobs=$(nproc) + if [ "$jobs" -gt 8 ]; then jobs=8; fi + echo "CMAKE_BUILD_PARALLEL_LEVEL=$jobs" >> "$GITHUB_ENV" + echo "CTEST_PARALLEL_LEVEL=$jobs" >> "$GITHUB_ENV" + + - name: Install Python 3 IDL compiler dependency + run: | + # The third-party cache contains C++ libraries, not Python code-generation dependencies. + apt-get update + apt-get install -y --no-install-recommends python3-yaml + /usr/bin/python3 -c 'import yaml' + + - name: Prepare runtime test clients + id: cmake_clients + working-directory: eloqdoc + shell: bash + run: | + apt-get install -y --no-install-recommends python3-venv + /usr/bin/python3 -m pip install --target build/cmake-ci/python 'pymongo==4.8.0' + source .github/scripts/common.sh + setup_python2 + python2 -c 'import yaml, pymongo, requests' + git clone --depth 1 --branch production \ + https://github.com/eloqdata/py-tpcc.git "$PY_TPCC_PATH" + + - name: Configure top-level CMake build + working-directory: eloqdoc + run: | + cmake -S . -B build/cmake-ci -G Ninja \ + -DPython3_EXECUTABLE=/usr/bin/python3 \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_JOB_POOLS=link_pool=1 \ + -DCMAKE_JOB_POOL_LINK=link_pool \ + -DBUILD_SHARED_LIBS=ON \ + -DELOQDOC_BUILD_TESTS=ON \ + -DELOQDOC_REGISTER_INTEGRATION_TESTS=ON \ + -DELOQDOC_TEST_CONNECTION_STRING=localhost:27017 \ + -DELOQ_THIRD_PARTY_PREFIX="$ELOQ_THIRD_PARTY_PREFIX" \ + -DELOQ_THIRD_PARTY_REQUIRED=ON \ + -DWITH_DATA_STORE=ELOQDSS_ELOQSTORE \ + -DWITH_LOG_STATE=ROCKSDB_CLOUD_S3 \ + -DWITH_LOG_SERVICE=ON \ + -DFORK_HM_PROCESS=ON \ + -DSTATISTICS=ON + + - name: Build server and all retained CMake tests + id: cmake_build + working-directory: eloqdoc + run: | + cmake --build build/cmake-ci \ + --target eloqdoc eloqdoc-tests + + - name: Run CMake unit and integration-startup tests + working-directory: eloqdoc + run: | + ctest --test-dir build/cmake-ci \ + --label-exclude '^integration$' --no-tests=error --output-on-failure + + - name: Run C++ integration suites against the CMake server + id: cmake_integration + if: ${{ !cancelled() && steps.cmake_build.outcome == 'success' && steps.cmake_clients.outcome == 'success' }} + working-directory: eloqdoc + shell: bash + run: | + PYTHONPATH="$PWD/build/cmake-ci/python" \ + bash .github/scripts/cmake_ci_runtime.sh build/cmake-ci integration + + - name: Build the legacy JavaScript test shell only + id: cmake_shell + if: ${{ !cancelled() && steps.cmake_build.outcome == 'success' && steps.cmake_clients.outcome == 'success' }} + working-directory: eloqdoc + shell: bash + run: | + BUILD_JOBS="$CMAKE_BUILD_PARALLEL_LEVEL" \ + bash .github/scripts/build_cmake_test_shell.sh \ + "$PWD/build/cmake-test-shell" "$PWD/build/cmake-test-client" + + - name: Run eloq_basic and eloq_core against the CMake server + id: cmake_jstests + if: ${{ !cancelled() && steps.cmake_shell.outcome == 'success' }} + working-directory: eloqdoc + shell: bash + run: | + ELOQDOC_TEST_SHELL="$PWD/build/cmake-test-client/bin/eloqdoc-cli" \ + PYTHONPATH="$PWD/build/cmake-ci/python" \ + bash .github/scripts/cmake_ci_runtime.sh build/cmake-ci jstests + + - name: Run TPCC against the CMake server + id: cmake_tpcc + if: ${{ !cancelled() && steps.cmake_build.outcome == 'success' && steps.cmake_clients.outcome == 'success' }} + working-directory: eloqdoc + shell: bash + run: | + PYTHONPATH="$PWD/build/cmake-ci/python" \ + bash .github/scripts/cmake_ci_runtime.sh build/cmake-ci tpcc + + - name: Preserve failed CMake runtime-test logs + if: ${{ failure() && (steps.cmake_integration.outcome == 'failure' || steps.cmake_jstests.outcome == 'failure' || steps.cmake_tpcc.outcome == 'failure') }} + uses: actions/upload-artifact@v4 + with: + name: cmake-runtime-eloqstore-s3-amd64-${{ github.run_attempt }} + path: | + eloqdoc/build/cmake-ci/runtime-diagnostics/ + eloqdoc/build/cmake-ci/jstests-results.json + eloqdoc/build/cmake-ci/Testing/Temporary/ + if-no-files-found: warn + retention-days: 7 + + - name: Exercise the normal server over the MongoDB wire protocol + id: cmake_smoke + if: ${{ !cancelled() && steps.cmake_build.outcome == 'success' }} + working-directory: eloqdoc + shell: bash + run: | + source .github/scripts/common.sh + trap 'stop_rustfs' EXIT + start_rustfs "$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" + PYTHONPATH="$PWD/build/cmake-ci/python" \ + cmake --build build/cmake-ci --target eloqdoc-server-smoke + + - name: Preserve failed CMake smoke-test logs + if: ${{ failure() && steps.cmake_smoke.outcome == 'failure' }} + uses: actions/upload-artifact@v4 + with: + name: cmake-smoke-eloqstore-s3-amd64-${{ github.run_attempt }} + path: | + eloqdoc/build/cmake-ci/smoke-diagnostics/ + /tmp/rustfs.log + if-no-files-found: warn + retention-days: 7 + ent-ci: needs: prime-third-party concurrency: @@ -58,19 +225,16 @@ jobs: strategy: fail-fast: false matrix: - build_type: [Debug, RelWithDebInfo] - arch: [amd64, arm64] + build_type: [RelWithDebInfo] + arch: [amd64] engine: - - id: rocksdb - data_store_type: ELOQDSS_ROCKSDB - log_state: ROCKSDB - id: rocks_s3 data_store_type: ELOQDSS_ROCKSDB_CLOUD_S3 log_state: ROCKSDB_CLOUD_S3 - id: eloqstore_s3 data_store_type: ELOQDSS_ELOQSTORE log_state: ROCKSDB_CLOUD_S3 - runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + runs-on: ubuntu-24.04 timeout-minutes: 240 container: image: eloqdata/ubuntu-dev:24.04 @@ -136,7 +300,8 @@ jobs: echo "JSTEST_TIMEOUT_SECONDS=7200" >> "$GITHUB_ENV" fi - - name: Build (${{ matrix.build_type }}, ${{ matrix.engine.id }}) + # gh_ci_entry.sh deliberately exercises the historical Python 2/SCons install-core path. + - name: Build with SCons (${{ matrix.build_type }}, ${{ matrix.engine.id }}) run: | bash ./eloqdoc/.github/scripts/gh_ci_entry.sh \ "$S3_ENDPOINT" \ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..7b98bfda13 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,390 @@ +cmake_minimum_required(VERSION 3.24) + +project(EloqDoc VERSION 4.0.3 LANGUAGES C CXX) + +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "Build type" FORCE) +endif() + +if(NOT CMAKE_GENERATOR MATCHES "Ninja") + message(FATAL_ERROR "EloqDoc's minimal build supports Ninja generators only. Use: cmake -G Ninja ...") +endif() +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "The minimal EloqDoc build currently supports Linux only") +endif() + +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS ON) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +option(ELOQDOC_BUILD_DATA_SUBSTRATE "Build and link the in-tree Data Substrate" ON) +option(ELOQDOC_BUILD_TESTS + "Build tests for the retained MongoDB runtime and Eloq Mongo module" OFF) +option(ELOQDOC_REGISTER_INTEGRATION_TESTS + "Register tests which require a running EloqDoc server with CTest" OFF) +option(ELOQDOC_ENABLE_ASAN "Build EloqDoc and Data Substrate with AddressSanitizer" OFF) +set(ELOQDOC_VERSION "4.0.3" CACHE STRING "MongoDB compatibility version reported by EloqDoc") +set(ELOQDOC_TEST_CONNECTION_STRING "localhost:27017" CACHE STRING + "MongoDB connection string used by C++ integration tests") +set(WITH_DATA_STORE "ELOQDSS_ELOQSTORE" CACHE STRING "Data Substrate storage backend") +set_property(CACHE WITH_DATA_STORE PROPERTY STRINGS + ROCKSDB ELOQDSS_ROCKSDB ELOQDSS_ROCKSDB_CLOUD_S3 + ELOQDSS_ROCKSDB_CLOUD_GCS ELOQDSS_ELOQSTORE) + +find_package(Python3 3.8 REQUIRED COMPONENTS Interpreter) +find_package(Threads REQUIRED) + +# PyYAML is needed by the IDL compiler in the selected Python 3 environment, even +# when the separately maintained C++ dependencies were restored from a cache. +execute_process( + COMMAND "${Python3_EXECUTABLE}" -c "import yaml" + RESULT_VARIABLE _eloqdoc_pyyaml_status + ERROR_VARIABLE _eloqdoc_pyyaml_error + OUTPUT_QUIET) +if(NOT _eloqdoc_pyyaml_status EQUAL 0) + message(FATAL_ERROR + "${Python3_EXECUTABLE} cannot import PyYAML (yaml), required for MongoDB IDL generation.\n" + "On Ubuntu with /usr/bin/python3, install python3-yaml. For a virtual environment, " + "install PyYAML into that environment and select it with -DPython3_EXECUTABLE=/path/to/python.\n" + "Installing PyYAML for the SCons Python 2 interpreter does not satisfy this dependency.\n" + "${_eloqdoc_pyyaml_error}") +endif() + +file(GLOB_RECURSE _eloqdoc_sconscript_files CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/src/*SConscript") +set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/generate_eloqdoc_sources.py" + ${_eloqdoc_sconscript_files}) + +set(_source_manifest "${CMAKE_CURRENT_BINARY_DIR}/eloqdoc-sources.cmake") +execute_process( + COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/generate_eloqdoc_sources.py" + --source-root "${CMAKE_CURRENT_SOURCE_DIR}/src" + --target-arch "${CMAKE_SYSTEM_PROCESSOR}" + --output "${_source_manifest}" + RESULT_VARIABLE _source_manifest_status + ERROR_VARIABLE _source_manifest_error) +if(NOT _source_manifest_status EQUAL 0) + message(FATAL_ERROR "Failed to select the minimal EloqDoc source graph:\n${_source_manifest_error}") +endif() +include("${_source_manifest}") + +list(LENGTH ELOQDOC_MONGO_TARGETS _mongo_target_count) +list(LENGTH ELOQDOC_MONGO_SOURCES _mongo_source_count) +list(LENGTH ELOQDOC_THIRD_PARTY_SOURCES _third_party_source_count) +list(LENGTH ELOQDOC_MONGO_TEST_IDS _mongo_test_count) +message(STATUS + "Minimal EloqDoc graph: ${_mongo_target_count} Mongo modules, " + "${_mongo_source_count} Mongo/Eloq sources, ${_third_party_source_count} vendored sources") +if(ELOQDOC_BUILD_TESTS) + message(STATUS "Single-node MongoDB test graph: ${_mongo_test_count} executables") +endif() + +# Generate the two configure-time headers which SCons used to synthesize. +set(mongo_config_altivec_vec_vbpermq_output_index "// #undef MONGO_CONFIG_ALTIVEC_VEC_VBPERMQ_OUTPUT_INDEX") +set(mongo_config_byte_order "#define MONGO_CONFIG_BYTE_ORDER 1234") +set(mongo_config_debug_build "// #undef MONGO_CONFIG_DEBUG_BUILD") +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(mongo_config_debug_build "#define MONGO_CONFIG_DEBUG_BUILD 1") +endif() +set(mongo_config_has_ssl_set_ecdh_auto "// #undef MONGO_CONFIG_HAS_SSL_SET_ECDH_AUTO") +set(mongo_config_have_execinfo_backtrace "#define MONGO_CONFIG_HAVE_EXECINFO_BACKTRACE 1") +set(mongo_config_have_fips_mode_set "// #undef MONGO_CONFIG_HAVE_FIPS_MODE_SET") +set(mongo_config_have_header_unistd_h "#define MONGO_CONFIG_HAVE_HEADER_UNISTD_H 1") +set(mongo_config_have_memset_s "// #undef MONGO_CONFIG_HAVE_MEMSET_S") +set(mongo_config_have_posix_monotonic_clock "#define MONGO_CONFIG_HAVE_POSIX_MONOTONIC_CLOCK 1") +set(mongo_config_have_pthread_setname_np "#define MONGO_CONFIG_HAVE_PTHREAD_SETNAME_NP 1") +set(mongo_config_have_std_make_unique "#define MONGO_CONFIG_HAVE_STD_MAKE_UNIQUE 1") +set(mongo_config_have_strnlen "#define MONGO_CONFIG_HAVE_STRNLEN 1") +set(mongo_config_max_extended_alignment "#define MONGO_CONFIG_MAX_EXTENDED_ALIGNMENT 64") +set(mongo_config_optimized_build "// #undef MONGO_CONFIG_OPTIMIZED_BUILD") +if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + set(mongo_config_optimized_build "#define MONGO_CONFIG_OPTIMIZED_BUILD 1") +endif() +set(mongo_config_ssl "// #undef MONGO_CONFIG_SSL") +set(mongo_config_ssl_has_asn1_any_definitions "// #undef MONGO_CONFIG_HAVE_ASN1_ANY_DEFINITIONS") +set(mongo_config_ssl_provider "// #undef MONGO_CONFIG_SSL_PROVIDER") +set(mongo_config_wiredtiger_enabled "// #undef MONGO_CONFIG_WIREDTIGER_ENABLED") +file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/generated/mongo/util") +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/config.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/generated/mongo/config.h" @ONLY) + +string(REGEX MATCH "^([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)$" _version_match "${ELOQDOC_VERSION}") +if(NOT _version_match) + message(FATAL_ERROR "ELOQDOC_VERSION must look like major.minor.patch[-extra]") +endif() +set(mongo_version "${ELOQDOC_VERSION}") +set(mongo_version_major "${CMAKE_MATCH_1}") +set(mongo_version_minor "${CMAKE_MATCH_2}") +set(mongo_version_patch "${CMAKE_MATCH_3}") +set(mongo_version_extra 0) +set(mongo_version_extra_str "${CMAKE_MATCH_4}") +execute_process( + COMMAND git rev-parse HEAD + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE mongo_git_hash + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) +if(NOT mongo_git_hash) + set(mongo_git_hash "unknown") +endif() +set(buildinfo_js_engine "mozjs") +set(buildinfo_allocator "system") +set(buildinfo_modules "{ \"eloq\" }") +set(buildinfo_environment_data "{}") +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/util/version_constants.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/generated/mongo/util/version_constants.h" @ONLY) + +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/MongoGeneratedSources.cmake") + +set(_eloqdoc_server_main "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/dbmain.cpp") +list(REMOVE_ITEM ELOQDOC_COMPILE_SOURCES "${_eloqdoc_server_main}") +set(_eloqdoc_mongo_core_sources) +set(_eloqdoc_eloq_storage_sources) +foreach(_source IN LISTS ELOQDOC_COMPILE_SOURCES) + if(_source MATCHES "/mongo/db/modules/eloq/") + list(APPEND _eloqdoc_eloq_storage_sources "${_source}") + else() + list(APPEND _eloqdoc_mongo_core_sources "${_source}") + endif() +endforeach() + +set(_third_party_absolute_sources) +foreach(_source IN LISTS ELOQDOC_THIRD_PARTY_SOURCES) + list(APPEND _third_party_absolute_sources "${CMAKE_CURRENT_SOURCE_DIR}/src/${_source}") +endforeach() + +add_library(eloqdoc-mongo-third-party OBJECT ${_third_party_absolute_sources}) +add_library(eloqdoc-mongo-runtime OBJECT ${_eloqdoc_mongo_core_sources}) +add_library(eloqdoc-mongo-eloq-storage OBJECT ${_eloqdoc_eloq_storage_sources}) +target_include_directories(eloqdoc-mongo-eloq-storage PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq") + +set(_mongo_include_directories + "${CMAKE_CURRENT_BINARY_DIR}/generated" + "${CMAKE_CURRENT_SOURCE_DIR}/src" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/asio-master/asio/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/boost-1.60.0" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/libstemmer_c/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/libstemmer_c/libstemmer" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/pcre-8.41" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/pcre-8.41/build_posix" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/s2" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/snappy-1.1.3" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/snappy-1.1.3/build_linux" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/timelib-2018.01alpha1" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/tomcrypt-1.18.1/src/headers" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/yaml-cpp-0.5.3/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/zlib-1.2.11") + +# Match src/third_party/icu4c-57.1/source/SConscript for the implementation and +# every consumer of its headers. Collation versions are persisted in index metadata. +set(_icu_root "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/icu4c-57.1/source") +set(_icu_definitions + UCONFIG_NO_BREAK_ITERATION=1 UCONFIG_NO_FORMATTING=1 + UCONFIG_NO_TRANSLITERATION=1 UCONFIG_NO_REGULAR_EXPRESSIONS=1 + U_CHARSET_IS_UTF8=1 U_DISABLE_RENAMING=1 + U_STATIC_IMPLEMENTATION=1 U_USING_ICU_NAMESPACE=0) + +# Match the pinned SpiderMonkey configuration used by SCons. These settings apply only to +# JavaScript implementation files, leaving MongoDB's own headers and compiler options intact. +set(_mozjs_root "${CMAKE_CURRENT_SOURCE_DIR}/src/third_party/mozjs-45") +list(APPEND _mongo_include_directories + "${_mozjs_root}/include" + "${_mozjs_root}/platform/${CMAKE_SYSTEM_PROCESSOR}/linux/include") +foreach(_source IN LISTS _third_party_absolute_sources ELOQDOC_COMPILE_SOURCES) + if(_source MATCHES "/third_party/mozjs-45/") + set_property(SOURCE "${_source}" APPEND PROPERTY INCLUDE_DIRECTORIES + "${_mozjs_root}/extract/js/src" "${_mozjs_root}/extract/mfbt" + "${_mozjs_root}/extract/intl/icu/source/common" "${_mozjs_root}/mongo_sources" + "${_mozjs_root}/platform/${CMAKE_SYSTEM_PROCESSOR}/linux/build") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + IMPL_MFBT JS_USE_CUSTOM_ALLOCATOR STATIC_JS_API=1 U_NO_DEFAULT_INCLUDE_UTF_HEADERS=1) + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_OPTIONS + -include js-confdefs.h -Wno-invalid-offsetof) + elseif(_source MATCHES "/mongo/scripting/mozjs/") + set_property(SOURCE "${_source}" APPEND PROPERTY INCLUDE_DIRECTORIES + "${_mozjs_root}/mongo_sources") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + JS_USE_CUSTOM_ALLOCATOR STATIC_JS_API=1) + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_OPTIONS + -include js-config.h -include js/RequiredDefines.h -Wno-invalid-offsetof) + endif() +endforeach() + +function(eloqdoc_configure_mongo_target target) + target_include_directories(${target} PRIVATE ${_mongo_include_directories}) + # ICU's source include directories also contain private names such as util.h. + # Search them after project/substrate headers, but before host system headers. + target_include_directories(${target} SYSTEM PRIVATE + "${_icu_root}/common" "${_icu_root}/i18n") + target_compile_definitions(${target} PRIVATE + _GNU_SOURCE + _XOPEN_SOURCE=700 + BOOST_SYSTEM_NO_DEPRECATED + BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS + ELOQDOC_STANDALONE + ELOQ_MODULE_ENABLED + ELOQ_MODULE_ELOQDOC + EXT_TX_PROC_ENABLED + ${_icu_definitions} + LTC_NO_PROTOTYPES + PCRE_STATIC) + target_compile_options(${target} PRIVATE + $<$:-fno-strict-aliasing> + $<$:-fno-strict-aliasing> + $<$:-fno-omit-frame-pointer> + $<$:-Wno-deprecated-declarations> + $<$:-Wno-nonnull> + $<$:-Wno-class-memaccess> + $<$:-Wno-interference-size> + $<$:-Wno-redundant-move>) +endfunction() + +foreach(_target IN ITEMS + eloqdoc-mongo-third-party + eloqdoc-mongo-runtime + eloqdoc-mongo-eloq-storage) + eloqdoc_configure_mongo_target(${_target}) +endforeach() + +set(_intel_decimal_definitions + DECIMAL_CALL_BY_REFERENCE=0 + DECIMAL_GLOBAL_ROUNDING=0 + DECIMAL_GLOBAL_EXCEPTION_FLAGS=0 + UNCHANGED_BINARY_STATUS_FLAGS=0 + USE_COMPILER_F128_TYPE=0 + USE_COMPILER_F80_TYPE=0 + USE_NATIVE_QUAD_TYPE=0 + LINUX=1 linux=1 gcc=1) +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|aarch64|ppc64le)$") + list(APPEND _intel_decimal_definitions efi2=1 EFI2=1) +endif() +foreach(_source IN LISTS _third_party_absolute_sources) + if(_source MATCHES "/IntelRDFPMathLib20U1/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + ${_intel_decimal_definitions}) + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_OPTIONS -w) + elseif(_source MATCHES "/pcre-8\\.41/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS HAVE_CONFIG_H) + elseif(_source MATCHES "/snappy-1\\.1\\.3/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS HAVE_CONFIG_H) + elseif(_source MATCHES "/timelib-2018\\.01alpha1/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + HAVE_GETTIMEOFDAY HAVE_STRING_H HAVE_DIRENT_H HAVE_SYS_TIME_H HAVE_UNISTD_H) + elseif(_source MATCHES "/zlib-1\\.2\\.11/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + HAVE_STDARG_H HAVE_UNISTD_H) + elseif(_source MATCHES "/s2/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS DEBUG_MODE=false) + elseif(_source MATCHES "/icu4c-57\\.1/source/") + if(_source MATCHES "/common/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + U_COMMON_IMPLEMENTATION=1) + elseif(_source MATCHES "/i18n/") + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_DEFINITIONS + U_I18N_IMPLEMENTATION=1) + endif() + set_property(SOURCE "${_source}" APPEND PROPERTY COMPILE_OPTIONS -Wno-sign-compare) + endif() +endforeach() + +add_dependencies(eloqdoc-mongo-runtime eloqdoc-generated-sources) +add_dependencies(eloqdoc-mongo-third-party eloqdoc-generated-sources) +add_dependencies(eloqdoc-mongo-eloq-storage eloqdoc-generated-sources) + +if(ELOQDOC_BUILD_DATA_SUBSTRATE) + set(ELOQ_MODULE_ENABLED ON CACHE BOOL "Register Data Substrate on brpc workers" FORCE) + set(EXT_TX_PROC_ENABLED ON CACHE BOOL "Drive TxProcessors from external workers" FORCE) + set(USE_ASAN "${ELOQDOC_ENABLE_ASAN}" CACHE BOOL "Data Substrate ASan" FORCE) + set(WITH_TESTS OFF CACHE BOOL "Data Substrate tests are maintained independently" FORCE) + set(ENABLE_TESTING OFF CACHE BOOL "Eloq metrics tests are maintained independently" FORCE) + set(WITH_UNIT_TESTS OFF CACHE BOOL "Data-store tests are maintained independently" FORCE) + set(WITH_EXAMPLE OFF CACHE BOOL "EloqStore examples are not part of EloqDoc" FORCE) + set(WITH_DB_STRESS OFF CACHE BOOL "EloqStore stress tools are not part of EloqDoc" FORCE) + add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq" eloq) + # The existing substrate build uses directory-scoped definitions rather than exporting + # all header/ABI switches. Share its actual settings with Mongo consumers without changing + # the independently maintained substrate build or duplicating its backend option mapping. + add_library(eloqdoc-data-substrate INTERFACE) + target_link_libraries(eloqdoc-data-substrate INTERFACE data_substrate) + # Some prebuilt dependencies (notably Abseil) do not carry their own RUNPATH. Use + # transitive RPATH lookup so launching from the build tree needs no LD_LIBRARY_PATH. + target_link_options(eloqdoc-data-substrate INTERFACE "LINKER:--disable-new-dtags") + target_compile_definitions(eloqdoc-data-substrate INTERFACE + $ + $<$:FAULT_INJECTOR>) + set(_data_substrate_target eloqdoc-data-substrate) +else() + add_library(eloqdoc-data-substrate-placeholder INTERFACE) + target_include_directories(eloqdoc-data-substrate-placeholder INTERFACE + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq/data_substrate" + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq/data_substrate/core/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq/data_substrate/tx_service/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq/data_substrate/eloq_metrics/include" + "${CMAKE_CURRENT_SOURCE_DIR}/src/mongo/db/modules/eloq/data_substrate/store_handler") + set(_data_substrate_target eloqdoc-data-substrate-placeholder) + message(WARNING + "ELOQDOC_BUILD_DATA_SUBSTRATE=OFF is for source-graph/compiler validation only; " + "the resulting executable cannot link") +endif() + +# MongoDB's Eloq integration includes Data Substrate headers directly. Propagate the selected +# target's public include paths and compile definitions while compiling the Mongo object library, +# not just while linking the final executable. +target_link_libraries(eloqdoc-mongo-runtime PRIVATE ${_data_substrate_target}) +target_link_libraries(eloqdoc-mongo-eloq-storage PRIVATE ${_data_substrate_target}) + +add_executable(eloqdoc + "${_eloqdoc_server_main}" + $ + $ + $) +if(TARGET host_manager) + # Substrate's default startup resolves this helper next to /proc/self/exe. Keep build-tree + # launches working as well as installs, where both executables already go into bin/. + set_target_properties(host_manager PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + target_link_options(host_manager PRIVATE "LINKER:--disable-new-dtags") + add_dependencies(eloqdoc host_manager) +endif() +eloqdoc_configure_mongo_target(eloqdoc) +set_target_properties(eloqdoc PROPERTIES + INSTALL_RPATH "$ORIGIN/../lib" + INSTALL_RPATH_USE_LINK_PATH TRUE) +target_link_libraries(eloqdoc PRIVATE + ${_data_substrate_target} + Threads::Threads + ${CMAKE_DL_LIBS} + m resolv rt) +target_link_options(eloqdoc PRIVATE -rdynamic -Wl,-z,now -Wl,-z,relro -Wl,-z,noexecstack) + +if(ELOQDOC_ENABLE_ASAN) + foreach(_target IN ITEMS + eloqdoc-mongo-third-party + eloqdoc-mongo-runtime + eloqdoc-mongo-eloq-storage + eloqdoc) + target_compile_options(${_target} PRIVATE -fsanitize=address) + target_link_options(${_target} PRIVATE -fsanitize=address) + endforeach() +endif() + +if(ELOQDOC_BUILD_TESTS) + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/MongoTests.cmake") +endif() + +install(TARGETS eloqdoc RUNTIME DESTINATION bin) + +add_custom_target(eloqdoc-source-manifest + COMMAND "${CMAKE_COMMAND}" -E echo + "${_mongo_target_count} selected modules, ${_mongo_source_count} Mongo/Eloq sources, ${_third_party_source_count} vendored sources" + DEPENDS "${_source_manifest}") diff --git a/cmake/MongoGeneratedSources.cmake b/cmake/MongoGeneratedSources.cmake new file mode 100644 index 0000000000..900a497942 --- /dev/null +++ b/cmake/MongoGeneratedSources.cmake @@ -0,0 +1,326 @@ +include_guard(GLOBAL) + +set(ELOQDOC_GENERATED_ROOT "${CMAKE_CURRENT_BINARY_DIR}/generated") +file(MAKE_DIRECTORY + "${ELOQDOC_GENERATED_ROOT}/mongo/base" + "${ELOQDOC_GENERATED_ROOT}/mongo/db/auth" + "${ELOQDOC_GENERATED_ROOT}/mongo/db/fts/unicode" + "${ELOQDOC_GENERATED_ROOT}/mongo/shell") + +option(ELOQDOC_CODEGEN_AT_CONFIGURE + "Generate any missing generated sources during configure, so a freshly \ +configured tree is immediately parseable by clangd and other tooling" ON) + +# Declare a generated source. This always emits the usual build-time rule, so +# editing an .idl/.err/.tpl input still regenerates on the next build. It also +# records the recipe so the configure-time bootstrap at the bottom of this file +# can produce outputs that do not exist yet. +function(_eloqdoc_codegen) + cmake_parse_arguments(ARG "" "MKDIR;DEPFILE;COMMENT" "OUTPUT;COMMAND;DEPENDS" ${ARGN}) + + set(_command_args) + if(ARG_MKDIR) + list(APPEND _command_args COMMAND "${CMAKE_COMMAND}" -E make_directory "${ARG_MKDIR}") + endif() + list(APPEND _command_args COMMAND ${ARG_COMMAND}) + + set(_optional_args) + if(ARG_DEPFILE) + list(APPEND _optional_args DEPFILE "${ARG_DEPFILE}") + endif() + if(ARG_COMMENT) + list(APPEND _optional_args COMMENT "${ARG_COMMENT}") + endif() + + add_custom_command( + OUTPUT ${ARG_OUTPUT} + ${_command_args} + DEPENDS ${ARG_DEPENDS} + ${_optional_args} + VERBATIM) + + get_property(_index GLOBAL PROPERTY ELOQDOC_CODEGEN_COUNT) + if(NOT _index) + set(_index 0) + endif() + set_property(GLOBAL PROPERTY ELOQDOC_CODEGEN_${_index}_OUTPUT "${ARG_OUTPUT}") + set_property(GLOBAL PROPERTY ELOQDOC_CODEGEN_${_index}_COMMAND "${ARG_COMMAND}") + set_property(GLOBAL PROPERTY ELOQDOC_CODEGEN_${_index}_MKDIR "${ARG_MKDIR}") + set_property(GLOBAL PROPERTY ELOQDOC_CODEGEN_${_index}_COMMENT "${ARG_COMMENT}") + math(EXPR _index "${_index} + 1") + set_property(GLOBAL PROPERTY ELOQDOC_CODEGEN_COUNT ${_index}) +endfunction() + +file(GLOB_RECURSE _eloqdoc_idl_compiler_sources CONFIGURE_DEPENDS + "${PROJECT_SOURCE_DIR}/scripts/buildscripts/idl/*.py") + +set(ELOQDOC_GENERATED_OUTPUTS) +set(_eloqdoc_all_source_requests ${ELOQDOC_MONGO_SOURCES}) +if(ELOQDOC_BUILD_TESTS) + list(APPEND _eloqdoc_all_source_requests ${ELOQDOC_MONGO_TEST_SUPPORT_SOURCES}) + foreach(_test_id IN LISTS ELOQDOC_MONGO_TEST_IDS) + list(APPEND _eloqdoc_all_source_requests + ${ELOQDOC_MONGO_TEST_${_test_id}_SOURCES}) + endforeach() +endif() +list(REMOVE_DUPLICATES _eloqdoc_all_source_requests) + +# Embed exactly the same collation data as SCons; do not rely on host ICU data files. +set(_icu_init_cpp "${ELOQDOC_GENERATED_ROOT}/mongo/util/icu_init.cpp") +if(CMAKE_C_BYTE_ORDER STREQUAL "BIG_ENDIAN") + set(_icu_data_name icudt57b.dat) +else() + set(_icu_data_name icudt57l.dat) +endif() +set(_icu_data "${PROJECT_SOURCE_DIR}/src/third_party/icu4c-57.1/source/mongo_sources/${_icu_data_name}") +_eloqdoc_codegen( + OUTPUT "${_icu_init_cpp}" + MKDIR "${ELOQDOC_GENERATED_ROOT}/mongo/util" + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/generate_icu_init.py" + "${PROJECT_SOURCE_DIR}/src/mongo/util/generate_icu_init_cpp.py" + -i "${_icu_data}" -o "${_icu_init_cpp}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/generate_icu_init.py" + "${PROJECT_SOURCE_DIR}/src/mongo/util/generate_icu_init_cpp.py" + "${_icu_data}" + COMMENT "Embedding pinned ICU 57.1 collation data") +list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_icu_init_cpp}") + +foreach(_source IN LISTS _eloqdoc_all_source_requests) + if(_source MATCHES "_gen\\.cpp$") + string(REGEX REPLACE "_gen\\.cpp$" ".idl" _idl_relative "${_source}") + set(_idl_input "${PROJECT_SOURCE_DIR}/src/${_idl_relative}") + if(NOT EXISTS "${_idl_input}") + message(FATAL_ERROR "No IDL input found for generated source ${_source}") + endif() + set(_generated_cpp "${ELOQDOC_GENERATED_ROOT}/${_source}") + string(REGEX REPLACE "\\.cpp$" ".h" _generated_h "${_generated_cpp}") + get_filename_component(_generated_dir "${_generated_cpp}" DIRECTORY) + _eloqdoc_codegen( + OUTPUT "${_generated_cpp}" "${_generated_h}" + MKDIR "${_generated_dir}" + COMMAND "${Python3_EXECUTABLE}" + "${PROJECT_SOURCE_DIR}/cmake/generate_idl.py" + --include "${PROJECT_SOURCE_DIR}/src" + --base_dir "${ELOQDOC_GENERATED_ROOT}" + --target_arch "${CMAKE_SYSTEM_PROCESSOR}" + --header "${_generated_h}" + --output "${_generated_cpp}" + --depfile "${_generated_cpp}.d" + "${_idl_input}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/generate_idl.py" + "${_idl_input}" + ${_eloqdoc_idl_compiler_sources} + DEPFILE "${_generated_cpp}.d" + COMMENT "Generating MongoDB IDL ${_idl_relative}") + list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_generated_cpp}" "${_generated_h}") + endif() +endforeach() + +set(_error_codes_h "${ELOQDOC_GENERATED_ROOT}/mongo/base/error_codes.h") +set(_error_codes_cpp "${ELOQDOC_GENERATED_ROOT}/mongo/base/error_codes.cpp") +_eloqdoc_codegen( + OUTPUT "${_error_codes_h}" "${_error_codes_cpp}" + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/generate_error_codes.py" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.err" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.tpl.h=${_error_codes_h}" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.tpl.cpp=${_error_codes_cpp}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/generate_error_codes.py" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.err" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.tpl.h" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.tpl.cpp" + COMMENT "Generating MongoDB error codes") +list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_error_codes_h}" "${_error_codes_cpp}") + +set(_action_type_h "${ELOQDOC_GENERATED_ROOT}/mongo/db/auth/action_type.h") +set(_action_type_cpp "${ELOQDOC_GENERATED_ROOT}/mongo/db/auth/action_type.cpp") +_eloqdoc_codegen( + OUTPUT "${_action_type_h}" "${_action_type_cpp}" + COMMAND "${Python3_EXECUTABLE}" + "${PROJECT_SOURCE_DIR}/cmake/generate_action_types.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/auth/generate_action_types.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/auth/action_types.txt" + "${_action_type_h}" "${_action_type_cpp}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/generate_action_types.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/auth/generate_action_types.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/auth/action_types.txt" + COMMENT "Generating MongoDB authorization action types") +list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_action_type_h}" "${_action_type_cpp}") + +set(_stop_words_h "${ELOQDOC_GENERATED_ROOT}/mongo/db/fts/stop_words_list.h") +set(_stop_words_cpp "${ELOQDOC_GENERATED_ROOT}/mongo/db/fts/stop_words_list.cpp") +file(GLOB _stop_word_inputs CONFIGURE_DEPENDS + "${PROJECT_SOURCE_DIR}/src/mongo/db/fts/stop_words_*.txt") +_eloqdoc_codegen( + OUTPUT "${_stop_words_h}" "${_stop_words_cpp}" + COMMAND "${Python3_EXECUTABLE}" + "${PROJECT_SOURCE_DIR}/cmake/run_legacy_text_generator.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/fts/generate_stop_words.py" + ${_stop_word_inputs} "${_stop_words_h}" "${_stop_words_cpp}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/run_legacy_text_generator.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/fts/generate_stop_words.py" + ${_stop_word_inputs} + COMMENT "Generating MongoDB text-search stop words") +list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_stop_words_h}" "${_stop_words_cpp}") + +set(_unicode_dir "${ELOQDOC_GENERATED_ROOT}/mongo/db/fts/unicode") +foreach(_kind IN ITEMS casefold delimiter_list diacritic_list) + set(_unicode_output "${_unicode_dir}/codepoints_${_kind}.cpp") + if(_kind STREQUAL "casefold") + set(_unicode_script "gen_casefold_map.py") + set(_unicode_input "${PROJECT_SOURCE_DIR}/src/third_party/unicode-8.0.0/CaseFolding.txt") + elseif(_kind STREQUAL "delimiter_list") + set(_unicode_script "gen_delimiter_list.py") + set(_unicode_input "${PROJECT_SOURCE_DIR}/src/third_party/unicode-8.0.0/PropList.txt") + else() + set(_unicode_script "gen_diacritic_list.py") + set(_unicode_input "${PROJECT_SOURCE_DIR}/src/third_party/unicode-8.0.0/PropList.txt") + endif() + _eloqdoc_codegen( + OUTPUT "${_unicode_output}" + COMMAND "${Python3_EXECUTABLE}" + "${PROJECT_SOURCE_DIR}/cmake/run_legacy_text_generator.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/fts/unicode/${_unicode_script}" + "${_unicode_input}" "${_unicode_output}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/run_legacy_text_generator.py" + "${PROJECT_SOURCE_DIR}/src/mongo/db/fts/unicode/${_unicode_script}" + "${PROJECT_SOURCE_DIR}/src/mongo/db/fts/unicode/gen_helper.py" + "${_unicode_input}" + COMMENT "Generating MongoDB Unicode ${_kind} table") + list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_unicode_output}") +endforeach() + +if("mongo/shell/mongo.cpp" IN_LIST ELOQDOC_MONGO_SOURCES) + set(_shell_error_codes "${ELOQDOC_GENERATED_ROOT}/mongo/shell/error_codes.js") + _eloqdoc_codegen( + OUTPUT "${_shell_error_codes}" + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/generate_error_codes.py" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.err" + "${PROJECT_SOURCE_DIR}/src/mongo/shell/error_codes.tpl.js=${_shell_error_codes}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/generate_error_codes.py" + "${PROJECT_SOURCE_DIR}/src/mongo/base/error_codes.err" + "${PROJECT_SOURCE_DIR}/src/mongo/shell/error_codes.tpl.js" + COMMENT "Generating shell error-code JavaScript") + + set(_mongo_js "${ELOQDOC_GENERATED_ROOT}/mongo/shell/mongo.cpp") + set(_mongo_js_inputs + assert.js bulk_api.js collection.js crud_api.js db.js explainable.js explain_query.js + mongo.js mr.js query.js session.js types.js utils_auth.js utils.js utils_sh.js) + set(_mongo_js_paths) + foreach(_file IN LISTS _mongo_js_inputs) + list(APPEND _mongo_js_paths "${PROJECT_SOURCE_DIR}/src/mongo/shell/${_file}") + endforeach() + _eloqdoc_codegen( + OUTPUT "${_mongo_js}" + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/generate_js.py" + "${PROJECT_SOURCE_DIR}/scons/site_tools/jstoh.py" + "${_mongo_js}" ${_mongo_js_paths} "${_shell_error_codes}" + DEPENDS + "${PROJECT_SOURCE_DIR}/cmake/generate_js.py" + "${PROJECT_SOURCE_DIR}/scons/site_tools/jstoh.py" + ${_mongo_js_paths} "${_shell_error_codes}" + COMMENT "Embedding MongoDB server JavaScript helpers") + list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_shell_error_codes}" "${_mongo_js}") +endif() + +if("mongo/scripting/mozjs/mongohelpers_js.cpp" IN_LIST ELOQDOC_MONGO_SOURCES) + set(_mongohelpers_js "${ELOQDOC_GENERATED_ROOT}/mongo/scripting/mozjs/mongohelpers_js.cpp") + _eloqdoc_codegen( + OUTPUT "${_mongohelpers_js}" + MKDIR "${ELOQDOC_GENERATED_ROOT}/mongo/scripting/mozjs" + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/generate_js.py" + "${PROJECT_SOURCE_DIR}/scons/site_tools/jstoh.py" + "${_mongohelpers_js}" "${PROJECT_SOURCE_DIR}/src/mongo/scripting/mozjs/mongohelpers.js" + DEPENDS "${PROJECT_SOURCE_DIR}/cmake/generate_js.py" + "${PROJECT_SOURCE_DIR}/scons/site_tools/jstoh.py" + "${PROJECT_SOURCE_DIR}/src/mongo/scripting/mozjs/mongohelpers.js" + COMMENT "Embedding MongoDB mongohelpers JavaScript") + list(APPEND ELOQDOC_GENERATED_OUTPUTS "${_mongohelpers_js}") +endif() + +function(eloqdoc_resolve_mongo_sources output) + set(_resolved) + foreach(_source IN LISTS ARGN) + if(_source MATCHES "_gen\\.cpp$" + OR _source STREQUAL "mongo/base/error_codes.cpp" + OR _source STREQUAL "mongo/db/auth/action_type.cpp" + OR _source STREQUAL "mongo/db/fts/stop_words_list.cpp" + OR _source MATCHES "^mongo/db/fts/unicode/codepoints_(casefold|delimiter_list|diacritic_list)\\.cpp$" + OR _source STREQUAL "mongo/shell/mongo.cpp" + OR _source STREQUAL "mongo/util/icu_init.cpp" + OR _source STREQUAL "mongo/scripting/mozjs/mongohelpers_js.cpp") + list(APPEND _resolved "${ELOQDOC_GENERATED_ROOT}/${_source}") + else() + list(APPEND _resolved "${PROJECT_SOURCE_DIR}/src/${_source}") + endif() + endforeach() + set(${output} "${_resolved}" PARENT_SCOPE) +endfunction() + +eloqdoc_resolve_mongo_sources(ELOQDOC_COMPILE_SOURCES ${ELOQDOC_MONGO_SOURCES}) + +set_source_files_properties(${ELOQDOC_GENERATED_OUTPUTS} PROPERTIES GENERATED TRUE) +add_custom_target(eloqdoc-generated-sources DEPENDS ${ELOQDOC_GENERATED_OUTPUTS}) + +# Configure-time bootstrap. +# +# The rules above only run when something is built. That leaves a freshly +# configured tree with an empty generated/ directory, so every translation unit +# that includes a generated header (mongo/base/error_codes.h, the IDL headers, +# ...) fails to parse. compile_commands.json already points at those headers, so +# clangd reports the whole tree as broken until the first build. +# +# Generate anything that is missing now. Outputs that already exist are left +# alone, so this costs nothing on repeat configures and never competes with the +# build-time rules, which remain responsible for regenerating stale outputs. +if(ELOQDOC_CODEGEN_AT_CONFIGURE) + get_property(_eloqdoc_codegen_count GLOBAL PROPERTY ELOQDOC_CODEGEN_COUNT) + if(NOT _eloqdoc_codegen_count) + set(_eloqdoc_codegen_count 0) + endif() + + set(_eloqdoc_bootstrapped 0) + if(_eloqdoc_codegen_count GREATER 0) + math(EXPR _eloqdoc_codegen_last "${_eloqdoc_codegen_count} - 1") + foreach(_i RANGE ${_eloqdoc_codegen_last}) + get_property(_outputs GLOBAL PROPERTY ELOQDOC_CODEGEN_${_i}_OUTPUT) + set(_needs_generating FALSE) + foreach(_output IN LISTS _outputs) + if(NOT EXISTS "${_output}") + set(_needs_generating TRUE) + break() + endif() + endforeach() + if(NOT _needs_generating) + continue() + endif() + + get_property(_command GLOBAL PROPERTY ELOQDOC_CODEGEN_${_i}_COMMAND) + get_property(_mkdir GLOBAL PROPERTY ELOQDOC_CODEGEN_${_i}_MKDIR) + get_property(_comment GLOBAL PROPERTY ELOQDOC_CODEGEN_${_i}_COMMENT) + if(_mkdir) + file(MAKE_DIRECTORY "${_mkdir}") + endif() + execute_process( + COMMAND ${_command} + RESULT_VARIABLE _codegen_status + ERROR_VARIABLE _codegen_error) + if(NOT _codegen_status EQUAL 0) + message(FATAL_ERROR + "Configure-time code generation failed (${_comment}):\n${_codegen_error}") + endif() + math(EXPR _eloqdoc_bootstrapped "${_eloqdoc_bootstrapped} + 1") + endforeach() + endif() + + if(_eloqdoc_bootstrapped GREATER 0) + message(STATUS + "EloqDoc: generated ${_eloqdoc_bootstrapped} missing source group(s) at configure time") + endif() +endif() diff --git a/cmake/MongoTests.cmake b/cmake/MongoTests.cmake new file mode 100644 index 0000000000..942e8bcd1a --- /dev/null +++ b/cmake/MongoTests.cmake @@ -0,0 +1,212 @@ +include_guard(GLOBAL) + +enable_testing() +add_test(NAME cmake/standalone-source-graph + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_source_graph.py") +set_tests_properties(cmake/standalone-source-graph PROPERTIES LABELS "cmake;unit") +add_test(NAME cmake/idl-dependencies + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_idl_dependencies.py") +set_tests_properties(cmake/idl-dependencies PROPERTIES LABELS "cmake;unit" TIMEOUT 60) +add_test(NAME cmake/scons-test-selection + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_scons_test_selection.py") +set_tests_properties(cmake/scons-test-selection PROPERTIES LABELS "cmake;unit" TIMEOUT 60) +add_test(NAME cmake/icu-data-generator + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_icu_generator.py") +set_tests_properties(cmake/icu-data-generator PROPERTIES LABELS "cmake;unit" TIMEOUT 60) +add_test(NAME cmake/server-smoke-config + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_server_smoke_config.py") +set_tests_properties(cmake/server-smoke-config PROPERTIES LABELS "cmake;unit" TIMEOUT 60) +add_test(NAME cmake/server-smoke-diagnostics + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_server_smoke_diagnostics.py") +set_tests_properties(cmake/server-smoke-diagnostics PROPERTIES LABELS "cmake;unit" TIMEOUT 60) +add_test(NAME cmake/server-fixture + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/test_server_fixture.py") +set_tests_properties(cmake/server-fixture PROPERTIES LABELS "cmake;unit" TIMEOUT 30) + +# Opt-in wire-protocol regression fixture; PyMongo is an explicit test-only dependency. +add_custom_target(eloqdoc-server-smoke + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/server_smoke.py" + --server "$" + --data-store "${WITH_DATA_STORE}" --log-state "${WITH_LOG_STATE}" + --diagnostics-dir "${CMAKE_CURRENT_BINARY_DIR}/smoke-diagnostics" + --ttl-shutdown-mode sleeping + COMMAND "${Python3_EXECUTABLE}" "${PROJECT_SOURCE_DIR}/cmake/tests/server_smoke.py" + --server "$" + --data-store "${WITH_DATA_STORE}" --log-state "${WITH_LOG_STATE}" + --diagnostics-dir "${CMAKE_CURRENT_BINARY_DIR}/smoke-diagnostics" + --ttl-shutdown-mode active + DEPENDS eloqdoc + USES_TERMINAL) + +# Keep the production objects in archives for tests. Unlike the server executable, a unit test +# should pull only the MongoDB objects it references; this avoids running unrelated global +# initializers and keeps individual test binaries reasonably small. +add_library(eloqdoc-mongo-runtime-test-archive STATIC EXCLUDE_FROM_ALL + $) +add_library(eloqdoc-mongo-third-party-test-archive STATIC EXCLUDE_FROM_ALL + $) +add_library(eloqdoc-mongo-eloq-test-archive STATIC EXCLUDE_FROM_ALL + $) + +# Compile each shared test helper once, but add only the helpers in a test's original SCons +# dependency closure to that executable. This preserves deliberate mock-vs-production choices. +set(_eloqdoc_mongo_test_support_object_targets) +foreach(_relative_source IN LISTS ELOQDOC_MONGO_TEST_SUPPORT_SOURCES) + string(SHA1 _source_id "${_relative_source}") + string(SUBSTRING "${_source_id}" 0 12 _source_id) + set(_support_target "eloqdoc-mongo-test-support-${_source_id}") + eloqdoc_resolve_mongo_sources(_support_source "${_relative_source}") + add_library(${_support_target} OBJECT EXCLUDE_FROM_ALL "${_support_source}") + eloqdoc_configure_mongo_target(${_support_target}) + add_dependencies(${_support_target} eloqdoc-generated-sources) + target_link_libraries(${_support_target} PRIVATE ${_data_substrate_target}) + if(ELOQDOC_ENABLE_ASAN) + target_compile_options(${_support_target} PRIVATE -fsanitize=address) + endif() + list(APPEND _eloqdoc_mongo_test_support_object_targets ${_support_target}) +endforeach() +add_custom_target(eloqdoc-mongo-test-support + DEPENDS ${_eloqdoc_mongo_test_support_object_targets}) + +# These objects implement string-named initializer groups used by every MongoDB test. They must be +# present directly in each executable (rather than discovered from a static archive), and object +# libraries let the complete test suite compile each common source only once. +add_library(eloqdoc-mongo-test-initializers OBJECT EXCLUDE_FROM_ALL + "${PROJECT_SOURCE_DIR}/src/mongo/base/init.cpp" + "${PROJECT_SOURCE_DIR}/src/mongo/base/shim.cpp" + "${PROJECT_SOURCE_DIR}/src/mongo/base/validate_locale.cpp" + "${PROJECT_SOURCE_DIR}/src/mongo/util/options_parser/startup_option_init.cpp") +add_library(eloqdoc-mongo-unit-test-main OBJECT EXCLUDE_FROM_ALL + "${PROJECT_SOURCE_DIR}/src/mongo/unittest/unittest_main.cpp") +add_library(eloqdoc-mongo-integration-test-main OBJECT EXCLUDE_FROM_ALL + "${PROJECT_SOURCE_DIR}/src/mongo/unittest/integration_test_main.cpp") +foreach(_support_target IN ITEMS + eloqdoc-mongo-test-initializers + eloqdoc-mongo-unit-test-main + eloqdoc-mongo-integration-test-main) + eloqdoc_configure_mongo_target(${_support_target}) + add_dependencies(${_support_target} eloqdoc-generated-sources) + target_link_libraries(${_support_target} PRIVATE ${_data_substrate_target}) + if(ELOQDOC_ENABLE_ASAN) + target_compile_options(${_support_target} PRIVATE -fsanitize=address) + endif() +endforeach() + +set(_eloqdoc_mongo_test_targets) +set(_eloqdoc_mongo_test_object_targets) +set(_eloqdoc_mongo_test_object_targets_without_eloq) +set(_eloqdoc_mongo_unit_test_count 0) +set(_eloqdoc_mongo_integration_test_count 0) + +foreach(_test_id IN LISTS ELOQDOC_MONGO_TEST_IDS) + set(_name "${ELOQDOC_MONGO_TEST_${_test_id}_NAME}") + set(_kind "${ELOQDOC_MONGO_TEST_${_test_id}_KIND}") + set(_requires_eloq "${ELOQDOC_MONGO_TEST_${_test_id}_REQUIRES_ELOQ}") + set(_relative_sources ${ELOQDOC_MONGO_TEST_${_test_id}_SOURCES}) + set(_relative_support_sources ${ELOQDOC_MONGO_TEST_${_test_id}_SUPPORT_SOURCES}) + eloqdoc_resolve_mongo_sources(_test_sources ${_relative_sources}) + set(_support_objects) + foreach(_relative_support_source IN LISTS _relative_support_sources) + string(SHA1 _support_source_id "${_relative_support_source}") + string(SUBSTRING "${_support_source_id}" 0 12 _support_source_id) + list(APPEND _support_objects + $) + endforeach() + + string(REPLACE "/" "-" _target_suffix "${_name}") + string(REPLACE "_" "-" _target_suffix "${_target_suffix}") + set(_target "eloqdoc-test-${_target_suffix}") + + if(_kind STREQUAL "unit") + set(_main_objects + $) + math(EXPR _eloqdoc_mongo_unit_test_count "${_eloqdoc_mongo_unit_test_count} + 1") + else() + set(_main_objects + $) + math(EXPR _eloqdoc_mongo_integration_test_count + "${_eloqdoc_mongo_integration_test_count} + 1") + endif() + set(_object_target "${_target}-objects") + add_library(${_object_target} OBJECT EXCLUDE_FROM_ALL ${_test_sources}) + eloqdoc_configure_mongo_target(${_object_target}) + add_dependencies(${_object_target} eloqdoc-generated-sources) + target_link_libraries(${_object_target} PRIVATE ${_data_substrate_target}) + if(ELOQDOC_ENABLE_ASAN) + target_compile_options(${_object_target} PRIVATE -fsanitize=address) + endif() + + add_executable(${_target} EXCLUDE_FROM_ALL + $ + ${_support_objects} + ${_main_objects} + $) + set_target_properties(${_target} PROPERTIES + OUTPUT_NAME "${_target_suffix}" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tests/mongo") + eloqdoc_configure_mongo_target(${_target}) + add_dependencies(${_target} eloqdoc-generated-sources) + set(_test_archives + eloqdoc-mongo-runtime-test-archive + eloqdoc-mongo-third-party-test-archive) + if(_requires_eloq) + list(APPEND _test_archives eloqdoc-mongo-eloq-test-archive) + endif() + string(JOIN "," _test_archive_group ${_test_archives}) + target_link_libraries(${_target} PRIVATE + "$" + ${_data_substrate_target} + Threads::Threads + ${CMAKE_DL_LIBS} + m resolv rt) + target_link_options(${_target} PRIVATE + -rdynamic -Wl,-z,now -Wl,-z,relro -Wl,-z,noexecstack) + if(ELOQDOC_ENABLE_ASAN) + target_compile_options(${_target} PRIVATE -fsanitize=address) + target_link_options(${_target} PRIVATE -fsanitize=address) + endif() + + if(_kind STREQUAL "unit") + add_test(NAME "${_name}" COMMAND ${_target}) + set_tests_properties("${_name}" PROPERTIES LABELS "mongo;unit" TIMEOUT 300 + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}") + if(_requires_eloq) + set_property(TEST "${_name}" APPEND PROPERTY LABELS "eloq") + endif() + else() + # Exercise integration-only initializer retention even without a running server. + add_test(NAME "${_name}/startup-options" COMMAND ${_target} --help) + set_tests_properties("${_name}/startup-options" PROPERTIES + LABELS "mongo;integration-startup" TIMEOUT 30 + PASS_REGULAR_EXPRESSION "--connectionString") + if(ELOQDOC_REGISTER_INTEGRATION_TESTS) + add_test(NAME "${_name}" COMMAND ${_target} + "--connectionString=${ELOQDOC_TEST_CONNECTION_STRING}") + set_tests_properties("${_name}" PROPERTIES + LABELS "mongo;integration" TIMEOUT 600 + RESOURCE_LOCK eloqdoc-integration-server) + endif() + endif() + + list(APPEND _eloqdoc_mongo_test_targets ${_target}) + list(APPEND _eloqdoc_mongo_test_object_targets ${_object_target}) + if(NOT _requires_eloq) + list(APPEND _eloqdoc_mongo_test_object_targets_without_eloq ${_object_target}) + endif() +endforeach() + +add_custom_target(eloqdoc-mongo-test-objects + DEPENDS ${_eloqdoc_mongo_test_object_targets}) +add_custom_target(eloqdoc-mongo-test-objects-without-eloq + DEPENDS ${_eloqdoc_mongo_test_object_targets_without_eloq}) +add_custom_target(eloqdoc-mongo-tests DEPENDS ${_eloqdoc_mongo_test_targets}) +add_custom_target(eloqdoc-tests DEPENDS ${_eloqdoc_mongo_test_targets}) + +message(STATUS + "MongoDB tests: ${_eloqdoc_mongo_unit_test_count} unit, " + "${_eloqdoc_mongo_integration_test_count} integration") +if(NOT ELOQDOC_REGISTER_INTEGRATION_TESTS AND _eloqdoc_mongo_integration_test_count GREATER 0) + message(STATUS + "MongoDB integration binaries will be built but not registered with CTest; " + "set ELOQDOC_REGISTER_INTEGRATION_TESTS=ON when a server fixture is available") +endif() diff --git a/cmake/generate_action_types.py b/cmake/generate_action_types.py new file mode 100644 index 0000000000..2c5a843009 --- /dev/null +++ b/cmake/generate_action_types.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Run MongoDB's Python 2 action-type generator for the CMake build.""" + +import sys +from pathlib import Path + + +def main(): + if len(sys.argv) < 2: + raise SystemExit("usage: generate_action_types.py SCRIPT [ARG ...]") + + script = Path(sys.argv[1]) + source = script.read_text(encoding="utf-8") + replacements = { + " print 'Duplicate actionType %s\\n' % actionType": + " print('Duplicate actionType %s\\n' % actionType)", + ' print "Usage: generate_action_types.py ' + '
"': + ' print("Usage: generate_action_types.py ' + '
")', + } + for old, new in replacements.items(): + if source.count(old) != 1: + raise RuntimeError("unexpected action-type generator format") + source = source.replace(old, new) + + sys.argv = sys.argv[1:] + globals_ = {"__file__": str(script), "__name__": "__main__"} + exec(compile(source, str(script), "exec"), globals_) + + +if __name__ == "__main__": + main() diff --git a/cmake/generate_eloqdoc_sources.py b/cmake/generate_eloqdoc_sources.py new file mode 100644 index 0000000000..abb39f29d9 --- /dev/null +++ b/cmake/generate_eloqdoc_sources.py @@ -0,0 +1,751 @@ +#!/usr/bin/env python3 +"""Extract the minimal EloqDoc source closure from MongoDB 4.0 SConscript files. + +The build itself is CMake/Ninja-only. This small configure-time translator keeps +the source manifest tied to MongoDB's fine-grained library declarations while +selecting the normal standalone server, the Eloq storage engine, and their +transitive dependencies. It never imports or executes SCons. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import os +import platform +from pathlib import Path, PurePosixPath +from typing import Any, Dict, Iterable, List, MutableMapping, Set + + +Target = Dict[str, Any] + +FORBIDDEN_TARGET_PREFIXES = ( + "mongo/mongos", + "mongo/db/repl/bgsync", + "mongo/db/repl/initial_syncer", + "mongo/db/repl/repl_coordinator_impl", + "mongo/db/repl/serveronly_repl", + "mongo/db/storage/mobile/storage_mobile", + "mongo/db/storage/mmap_v1/storage_mmapv1", + "mongo/db/storage/wiredtiger/", + "third_party/wiredtiger/", + "mongo/embedded/embedded", + "mongo/embedded/mongoed", + "mongo/db/s/balancer", + "mongo/db/s/commands_db_s", + "mongo/db/s/sharding_catalog_manager", + "mongo/s/commands/", + "mongo/s/sharding_legacy_api", +) + +EXCLUDED_TEST_TARGET_PREFIXES = ( + "mongo/db/repl/", + "mongo/db/s/", + "mongo/db/storage/mmap_v1/", + "mongo/db/storage/mobile/", + "mongo/db/storage/wiredtiger/", + "mongo/s/", + "mongo/shell/", +) + +# Distributed-only suites outside the repl/ and sharding directories. Their libraries +# may still supply shared types to the standalone server; that does not make these +# feature-specific tests part of EloqDoc validation. SCons targets remain unchanged. +EXCLUDED_DISTRIBUTED_TEST_TARGETS = { + "mongo/client/dbclient_rs_test": "replica-set client routing/failover", + "mongo/client/replica_set_monitor_test": "replica-set topology monitoring", + "mongo/db/keys_collection_document_test": "cluster-time signing keys", + "mongo/db/operation_time_tracker_test": "operation times from remote MongoDB servers", + "mongo/db/time_proof_service_test": "cluster-time signing/verification", + "mongo/rpc/config_server_metadata_test": "config-server metadata", + "mongo/rpc/oplog_query_metadata_test": "replication oplog-query metadata", + "mongo/rpc/repl_set_metadata_test": "replica-set metadata", +} + +SOURCE_REPLACEMENTS = { + "mongo/db/ftdc/ftdc_system_stats_${TARGET_OS}.cpp": "mongo/db/ftdc/ftdc_system_stats_linux.cpp", + "mongo/db/storage/mmap_v1/mmap_${TARGET_OS_FAMILY}.cpp": "mongo/db/storage/mmap_v1/mmap_posix.cpp", + "mongo/crypto/sha_block_${MONGO_CRYPTO}.cpp": "mongo/crypto/sha_block_tom.cpp", + "mongo/db/storage/storage_engine_lock_file_${TARGET_OS_FAMILY}.cpp": + "mongo/db/storage/storage_engine_lock_file_posix.cpp", + "mongo/platform/shared_library_${TARGET_OS_FAMILY}.cpp": + "mongo/platform/shared_library_posix.cpp", + "mongo/platform/stack_locator_${TARGET_OS}.cpp": "mongo/platform/stack_locator_linux.cpp", + "mongo/util/processinfo_${TARGET_OS}.cpp": "mongo/util/processinfo_linux.cpp", + "mongo/util/stacktrace_${TARGET_OS_FAMILY}.cpp": "mongo/util/stacktrace_posix.cpp", +} + +GENERATED_WITHOUT_SOURCE_FILE = { + "mongo/base/error_codes.cpp", + "mongo/db/auth/action_type.cpp", + "mongo/db/fts/stop_words_list.cpp", + "mongo/db/fts/unicode/codepoints_casefold.cpp", + "mongo/db/fts/unicode/codepoints_delimiter_list.cpp", + "mongo/db/fts/unicode/codepoints_diacritic_list.cpp", + "mongo/shell/mongo.cpp", + "mongo/scripting/mozjs/mongohelpers_js.cpp", + "mongo/util/icu_init.cpp", +} + + +def _flatten(value: Any) -> List[str]: + if isinstance(value, list): + result: List[str] = [] + for item in value: + result.extend(_flatten(item)) + return result + return [value] if isinstance(value, str) else [] + + +class StaticEvaluator: + """Evaluate the literal subset used for SConscript source/dependency lists.""" + + def __init__(self, variables: MutableMapping[str, Any]): + self.variables = variables + + def eval(self, node: ast.AST | None, locals_: Dict[str, Any] | None = None) -> Any: + if node is None: + return [] + local_values = locals_ or {} + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, (ast.List, ast.Tuple)): + return [self.eval(item, local_values) for item in node.elts] + if isinstance(node, ast.Name): + return local_values.get(node.id, self.variables.get(node.id, [])) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not bool(self.eval(node.operand, local_values)) + if isinstance(node, ast.BinOp): + left = self.eval(node.left, local_values) + right = self.eval(node.right, local_values) + if isinstance(node.op, ast.Add): + if isinstance(left, list) or isinstance(right, list): + return _flatten(left) + _flatten(right) + return left + right + if isinstance(node.op, ast.Mod): + try: + return left % right + except (TypeError, ValueError): + return [] + if isinstance(node, ast.IfExp): + branch = node.body if self.eval(node.test, local_values) else node.orelse + return self.eval(branch, local_values) + if isinstance(node, ast.Compare) and len(node.ops) == 1: + left = self.eval(node.left, local_values) + right = self.eval(node.comparators[0], local_values) + if isinstance(node.ops[0], ast.Eq): + return left == right + if isinstance(node.ops[0], ast.NotEq): + return left != right + if isinstance(node.ops[0], ast.In): + return left in right + if isinstance(node.ops[0], ast.NotIn): + return left not in right + if isinstance(node, ast.BoolOp): + values = [bool(self.eval(item, local_values)) for item in node.values] + return all(values) if isinstance(node.op, ast.And) else any(values) + if isinstance(node, ast.Subscript): + value = self.eval(node.value, local_values) + index = self.eval(node.slice, local_values) + try: + return value[index] + except (IndexError, KeyError, TypeError): + return [] + if isinstance(node, ast.ListComp) and len(node.generators) == 1: + generator = node.generators[0] + if not isinstance(generator.target, ast.Name): + return [] + result = [] + for item in _flatten(self.eval(generator.iter, local_values)): + nested = dict(local_values) + nested[generator.target.id] = item + result.append(self.eval(node.elt, nested)) + return result + if isinstance(node, ast.Call): + if isinstance(node.func, ast.Name) and node.func.id == "use_system_version_of_library": + # Match SCons' pinned dependencies, including ICU: its version is stored in + # index collation specifications, so using the host's ICU changes compatibility. + return False + if isinstance(node.func, ast.Attribute): + method = node.func.attr + if method == "TargetOSIs": + return any(self.eval(arg, local_values) in ("linux", "posix") for arg in node.args) + if method == "ToolchainIs": + return any(self.eval(arg, local_values) in ("GCC", "gcc") for arg in node.args) + if method == "Idlc": + source = Path(str(self.eval(node.args[0], local_values))) + generated_base = source.with_suffix("").as_posix() + "_gen" + # SCons returns [generated_cpp, generated_header]; Mongo's declarations select + # element zero. Preserve that shape so the surrounding AST subscript works. + return [generated_base + ".cpp", generated_base + ".h"] + if method == "WindowsResourceFile": + return [] + return [] + + +def _active_statements( + statements: Iterable[ast.stmt], evaluator: StaticEvaluator +) -> Iterable[ast.stmt]: + """Yield statements from branches which are active in the Linux build profile.""" + for statement in statements: + if isinstance(statement, ast.If): + branch = statement.body if evaluator.eval(statement.test) else statement.orelse + yield from _active_statements(branch, evaluator) + else: + yield statement + + +def _collect_variables(tree: ast.Module, target_arch: str) -> Dict[str, Any]: + variables: Dict[str, Any] = { + "debugBuild": False, + "env": { + "MONGO_ALLOCATOR": "system", + "MONGO_BUILD_SASL_CLIENT": True, + "MONGO_CRYPTO": "tom", + "MONGO_HAVE_LIBMONGOC": False, + "MONGO_MODULES": ["eloq"], + "MODULE_BANNERS": [], + "TARGET_ARCH": target_arch, + "TARGET_OS": "linux", + }, + "free_monitoring": "off", + "mmapv1": False, + "serverJs": True, + "usemozjs": True, + "wiredtiger": False, + } + evaluator = StaticEvaluator(variables) + # All source variables in this MongoDB branch are literal lists declared before use. Repeating + # the walk resolves lists which refer to an earlier list. Walking only the selected branch is + # important: several scripts assign Linux and Windows dependencies to the same variable. + for _ in range(3): + for statement in _active_statements(tree.body, evaluator): + for node in ast.walk(statement): + if isinstance(node, ast.If): + # Nested conditionals are handled by _active_statements, not ast.walk. + continue + if not isinstance(node, ast.Assign) or len(node.targets) != 1: + continue + target = node.targets[0] + if isinstance(target, ast.Name): + value = evaluator.eval(node.value) + if value != []: + variables[target.id] = value + return variables + + +def _canonical_dependency(base: str, dependency: str) -> str: + if dependency.startswith("$BUILD_DIR/"): + return dependency[len("$BUILD_DIR/") :] + if dependency.startswith("#/"): + return dependency[2:] + return os.path.normpath((PurePosixPath(base) / dependency).as_posix()) + + +def _canonical_source(base: str, source: str) -> str: + if source.startswith("$BUILD_DIR/"): + return source[len("$BUILD_DIR/") :] + return (PurePosixPath(base) / source).as_posix() + + +def load_targets(source_root: Path, target_arch: str = platform.machine()) -> Dict[str, Target]: + targets: Dict[str, Target] = {} + for script in source_root.rglob("SConscript"): + tree = ast.parse(script.read_text(encoding="utf-8"), filename=str(script)) + evaluator = StaticEvaluator(_collect_variables(tree, target_arch)) + base = script.parent.relative_to(source_root).as_posix() + for statement in _active_statements(tree.body, evaluator): + for node in ast.walk(statement): + if isinstance(node, ast.If): + # Nested conditionals are handled by _active_statements, not ast.walk. + continue + if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute): + continue + kind = node.func.attr + if kind not in ("Library", "Program", "CppUnitTest", "CppIntegrationTest"): + continue + keywords = {item.arg: item.value for item in node.keywords} + target_node = keywords.get("target", node.args[0] if node.args else None) + source_node = keywords.get("source", node.args[1] if len(node.args) > 1 else None) + names = _flatten(evaluator.eval(target_node)) + sources = _flatten(evaluator.eval(source_node)) + dependencies: List[str] = [] + for field in ("LIBDEPS", "LIBDEPS_PRIVATE", "LIBDEPS_INTERFACE"): + dependencies.extend(_flatten(evaluator.eval(keywords.get(field)))) + if kind == "CppUnitTest": + dependencies.append("$BUILD_DIR/mongo/unittest/unittest_main") + elif kind == "CppIntegrationTest": + dependencies.append("$BUILD_DIR/mongo/unittest/integration_test_main") + for name in names: + canonical_name = (PurePosixPath(base) / name).as_posix() + record = targets.setdefault( + canonical_name, {"sources": [], "dependencies": [], "kind": kind} + ) + record["sources"].extend(_canonical_source(base, item) for item in sources) + record["dependencies"].extend( + _canonical_dependency(base, item) for item in dependencies + ) + return targets + + +def select_minimal_graph(targets: Dict[str, Target], source_root: Path | None = None, + target_arch: str = platform.machine()) -> Set[str]: + # Keep db.cpp, ServiceEntryPointMongod, authentication, maintenance and JavaScript. + # Only the replication/sharding boundaries and alternative engines are removed. + distributed = { + "mongo/db/repl/bgsync", "mongo/db/repl/oplog_buffer_blocking_queue", + "mongo/db/repl/oplog_buffer_collection", "mongo/db/repl/oplog_buffer_proxy", + "mongo/db/repl/repl_coordinator_impl", "mongo/db/repl/rs_rollback", + "mongo/db/repl/rslog", "mongo/db/repl/serveronly_repl", + "mongo/db/repl/oplog_application", "mongo/db/repl/topology_coordinator", + "mongo/db/repl/repl_set_commands", "mongo/db/s/commands_db_s", + "mongo/db/s/balancer", "mongo/db/s/op_observer_sharding_impl", + "mongo/db/s/sharding_catalog_manager", "mongo/s/sharding_legacy_api", + "mongo/s/catalog/sharding_catalog_client_impl", + "mongo/s/commands/shared_cluster_commands", + "mongo/db/storage/mmap_v1/storage_mmapv1", + "mongo/db/storage/devnull/storage_devnull", + "mongo/db/storage/ephemeral_for_test/storage_ephemeral_for_test", + } + for name in ("mongo/mongodmain", "mongo/db/serveronly", "mongo/db/commands/mongod", + "mongo/db/commands/mongod_fsync", "mongo/db/commands/servers"): + targets[name]["dependencies"] = [ + dep for dep in targets[name]["dependencies"] if dep not in distributed + ] + targets["mongo/mongodmain"]["dependencies"].extend([ + "mongo/db/modules/eloq/storage_eloq", "mongo/db/repl/repl_coordinator_standalone", + "mongo/db/op_observer_impl", "mongo/db/repl/replication_info", + ]) + # This target also registers the standalone isMaster handshake and diagnostics. It does + # not use OplogReader; the old dependency must not retain a replica-set oplog client. + targets["mongo/db/repl/replication_info"]["dependencies"] = [ + dep for dep in targets["mongo/db/repl/replication_info"]["dependencies"] + if dep != "mongo/db/repl/oplogreader" + ] + # The interface supplies shared write-concern constants without the catalog client runtime. + targets["mongo/db/keys_collection_client_direct"]["dependencies"].append( + "mongo/s/catalog/sharding_catalog_client") + targets["mongo/db/repl/repl_coordinator_standalone"] = { + "sources": ["mongo/db/repl/replication_coordinator_standalone.cpp"], + "dependencies": ["mongo/db/repl/repl_coordinator_interface", + "mongo/db/repl/repl_settings", "mongo/db/global_settings"], + "kind": "Library", + } + targets["mongo/db/standalone_runtime_test"] = { + "sources": ["mongo/db/standalone_runtime_test.cpp"], + "dependencies": ["mongo/db/repl/repl_coordinator_standalone", + "mongo/unittest/unittest_main"], + "kind": "CppUnitTest", + } + for name in ("mongo/mongodmain", "mongo/db/serveronly"): + targets[name]["dependencies"] = [ + "mongo/db/s/sharding_runtime_d_embedded" + if dep == "mongo/db/s/sharding_runtime_d" else dep + for dep in targets[name]["dependencies"] + ] + # Replica-set-only commands must not register on this standalone server. + targets["mongo/db/commands/mongod"]["sources"] = [ + src for src in targets["mongo/db/commands/mongod"]["sources"] + if Path(src).name not in {"resize_oplog.cpp", "oplog_note.cpp", "dbcheck.cpp"} + ] + targets["mongo/db/commands/mongod"]["dependencies"] = [ + dep for dep in targets["mongo/db/commands/mongod"]["dependencies"] + if dep != "mongo/db/repl/dbcheck" + ] + # Substitute the small factory at the link boundary instead of branching shared code. + targets["mongo/db/logical_session_cache_factory_mongod"]["sources"] = [ + "mongo/db/logical_session_cache_factory_standalone.cpp", + ] + targets["mongo/db/logical_session_cache_factory_mongod"]["dependencies"] = [ + "mongo/db/logical_session_cache_impl", "mongo/db/service_liaison_mongod", + "mongo/db/sessions_collection_standalone", + ] + + # The common request path needs CollectionShardingState, not the shard-server runtime. + # Reuse MongoDB's unsharded metadata implementation without its embedded application. + entry_point = targets["mongo/db/service_entry_point_common"] + entry_point["dependencies"] = [ + ( + "mongo/db/s/sharding_runtime_d_embedded" + if item == "mongo/db/s/sharding_runtime_d" + else item + ) + for item in entry_point["dependencies"] + ] + + # Keep local Pipeline functionality, but omit the separately compiled sharding entry points, + # distributed planner, and remote cursor implementation. No rejection stubs are needed: + # pipeline.cpp no longer references the distributed planner. + pipeline = targets["mongo/db/pipeline/pipeline"] + pipeline["sources"] = [ + item for item in pipeline["sources"] + if item + not in { + "mongo/db/pipeline/pipeline_sharded.cpp", + "mongo/db/pipeline/cluster_aggregation_planner.cpp", + "mongo/db/pipeline/document_source_internal_split_pipeline.cpp", + "mongo/db/pipeline/document_source_merge_cursors.cpp", + } + ] + # pipeline.h exposes AsyncResultsMergerParams even in the local-only planner. Retain the IDL + # value type (but not the async-results-merger runtime) so clean builds generate its header and + # provide any serialization symbols used by the shared pipeline API. + pipeline["sources"].append("mongo/s/query/async_results_merger_params_gen.cpp") + pipeline["dependencies"] = [ + item + for item in pipeline["dependencies"] + if item != "mongo/s/query/async_results_merger" + ] + + # dbcommands.cpp consumes the generated profile request type. Full mongod reaches it through + # the mongod command bundle, while the embedded command bundle omitted that edge. + targets["mongo/db/commands/standalone"]["dependencies"].append( + "mongo/db/commands/profile_common" + ) + + # catalog_impl has no Balancer symbol references. This legacy LIBDEPS edge exists to retain + # static initializers in full mongod; the single-node executable must not retain them. + catalog_impl = targets["mongo/db/catalog/catalog_impl"] + catalog_impl["dependencies"] = [ + item for item in catalog_impl["dependencies"] if item != "mongo/db/s/balancer" + ] + + # Standalone startup does not initialize a sharded logical-time key client. + keys_manager = targets["mongo/db/keys_collection_manager"] + keys_manager["dependencies"] = [ + item + for item in keys_manager["dependencies"] + if item != "mongo/db/keys_collection_client_sharded" + ] + + # The SpiderMonkey SConscript appends generated unified translation units with Glob(). + # Keep that explicit platform source set without executing SCons. + mozjs = targets["third_party/mozjs-45/mozjs"] + if target_arch == "x86_64": + mozjs["sources"].append( + "third_party/mozjs-45/extract/js/src/jit/x86-shared/Disassembler-x86-shared.cpp") + mozjs_root = source_root or Path(__file__).resolve().parents[1] / "src" + mozjs_platform = mozjs_root / "third_party/mozjs-45/platform" / target_arch / "linux/build" + if not mozjs_platform.is_dir(): + raise RuntimeError(f"No pinned SpiderMonkey configuration for {target_arch}/linux") + mozjs["sources"].extend( + path.relative_to(mozjs_root).as_posix() + for path in sorted(mozjs_platform.glob("*.cpp")) + ) + + # TLS is excluded from this single-node profile, so use MongoDB's small vendored TomCrypt + # implementation for SCRAM hashes rather than compiling an SSL provider. + targets["mongo/crypto/sha_block_${MONGO_CRYPTO}"] = { + "sources": ["mongo/crypto/sha_block_tom.cpp"], + "dependencies": [ + "mongo/base", + "mongo/crypto/sha1_block", + "mongo/crypto/sha256_block", + "third_party/shim_tomcrypt", + ], + "kind": "Library", + } + + # SCons carries interface dependencies on cloned environments. Recreate the subset used by + # this build explicitly; these are ordinary vendored libraries, not SCons runtime behavior. + interface_dependencies = { + "third_party/shim_asio": ["third_party/asio-master/asio"], + "third_party/shim_boost": [ + "third_party/boost-1.60.0/boost_program_options", + "third_party/boost-1.60.0/boost_filesystem", + "third_party/boost-1.60.0/boost_system", + "third_party/boost-1.60.0/boost_iostreams", + ], + "third_party/shim_intel_decimal128": [ + "third_party/IntelRDFPMathLib20U1/intel_decimal128" + ], + "third_party/shim_icu": ["third_party/icu4c-57.1/source/icu_i18n"], + "third_party/shim_pcrecpp": ["third_party/pcre-8.41/pcrecpp"], + "third_party/shim_mozjs": ["third_party/mozjs-45/mozjs"], + "third_party/shim_snappy": ["third_party/snappy-1.1.3/snappy"], + "third_party/shim_stemmer": ["third_party/libstemmer_c/stemmer"], + "third_party/shim_timelib": ["third_party/timelib-2018.01alpha1/timelib"], + "third_party/shim_tomcrypt": ["third_party/tomcrypt-1.18.1/tomcrypt"], + "third_party/shim_yaml": ["third_party/yaml-cpp-0.5.3/yaml"], + "third_party/shim_zlib": ["third_party/zlib-1.2.11/zlib"], + } + for target, dependencies in interface_dependencies.items(): + targets[target]["dependencies"].extend(dependencies) + + selected: Set[str] = set() + pending = ["mongo/eloqdoc"] + while pending: + name = pending.pop() + if name in selected: + continue + if name not in targets: + raise RuntimeError(f"Minimal EloqDoc graph references unknown target: {name}") + selected.add(name) + pending.extend(targets[name]["dependencies"]) + + bad = sorted( + name + for name in selected + if any(name.startswith(item) for item in FORBIDDEN_TARGET_PREFIXES) + ) + if bad: + raise RuntimeError("Forbidden distributed/alternate-engine targets selected: " + ", ".join(bad)) + return selected + + +def _dependency_closure(targets: Dict[str, Target], roots: Iterable[str]) -> Set[str] | None: + selected: Set[str] = set() + pending = list(roots) + while pending: + name = pending.pop() + if name in selected: + continue + if name not in targets: + return None + selected.add(name) + pending.extend(targets[name]["dependencies"]) + return selected + + +def _normalized_sources(sources: Iterable[str]) -> Set[str]: + normalized = {SOURCE_REPLACEMENTS.get(source, source) for source in sources} + normalized.difference_update( + { + "mongo/util/exception_filter_win32.cpp", + "mongo/util/signal_win32.cpp", + "mongo/util/winutil.cpp", + "third_party/shim_intel_decimal128.cppshim_intel_decimal128.cpp", + } + ) + if "third_party/shim_intel_decimal128.cppshim_intel_decimal128.cpp" in sources: + normalized.add("third_party/shim_intel_decimal128.cpp") + return {source for source in normalized if source.endswith((".c", ".cc", ".cpp"))} + + +def select_minimal_tests( + targets: Dict[str, Target], production_targets: Set[str] +) -> tuple[List[Dict[str, Any]], Set[str]]: + """Select tests for the retained single-node modules and their shared test helpers.""" + # Production drops alternative engine registrations, but the MongoD test fixture + # explicitly starts ephemeralForTest. Restore that edge only in the test graph. + targets = dict(targets) + fixture_name = "mongo/db/service_context_d_test_fixture" + targets[fixture_name] = { + **targets[fixture_name], + "dependencies": targets[fixture_name]["dependencies"] + [ + "mongo/db/storage/ephemeral_for_test/storage_ephemeral_for_test"], + } + production_sources = _normalized_sources( + source for name in production_targets for source in targets[name]["sources"] + ) + production_directories = { + PurePosixPath(source).parent.as_posix() for source in production_sources + } + tests: List[Dict[str, Any]] = [] + support_targets: Set[str] = set() + # String-named initialization edges cannot pull an otherwise unreferenced object out of a + # static archive. Retain these providers directly, but only for their original test closure. + initializer_sources = { + # These providers run through named initializers, not references from the test main. + # Retain them only when their original SCons dependency closure includes them. + "mongo/util/options_parser/options_parser_init.cpp", + "mongo/util/version_impl.cpp", + "mongo/util/icu_init.cpp", + "mongo/db/auth/authorization_manager_global.cpp", + # The global constructor calls these factories through MONGO_REGISTER_SHIM, not + # through a linker reference to the implementation classes. Retain both providers + # for query/service-context tests as well as tests that use the classes directly. + "mongo/db/auth/authorization_manager_impl.cpp", + "mongo/db/auth/authorization_session_impl.cpp", + "mongo/db/catalog/database_holder_impl.cpp", + "mongo/db/catalog/database_impl.cpp", + "mongo/db/catalog/collection_impl.cpp", + "mongo/db/catalog/collection_info_cache_impl.cpp", + "mongo/db/catalog/index_catalog_entry_impl.cpp", + "mongo/db/catalog/index_catalog_impl.cpp", + "mongo/db/catalog/index_create_impl.cpp", + } + # Aggregation factories are looked up by stage/accumulator name, not by a C++ symbol. + # As with SCons' library retention, keep their implementation objects when the original + # test closure includes them; otherwise authorization/parser tests silently lose stages. + initializer_sources.update( + source for source in production_sources + if source.startswith(("mongo/db/pipeline/document_source_", + "mongo/db/pipeline/accumulator_", + "mongo/db/pipeline/granularity_rounder_")) + ) + + for name, target in sorted(targets.items()): + if target["kind"] not in ("CppUnitTest", "CppIntegrationTest"): + continue + if name in EXCLUDED_DISTRIBUTED_TEST_TARGETS: + continue + if any(name.startswith(prefix) for prefix in EXCLUDED_TEST_TARGET_PREFIXES): + continue + + main_target = ( + "mongo/unittest/unittest_main" + if target["kind"] == "CppUnitTest" + else "mongo/unittest/integration_test_main" + ) + explicit_dependencies = [ + dependency for dependency in target["dependencies"] if dependency != main_target + ] + dependency_closure = _dependency_closure(targets, target["dependencies"]) + related_closure = _dependency_closure(targets, explicit_dependencies) or set() + if dependency_closure is None: + continue + if any( + dependency.startswith(prefix) + for dependency in dependency_closure + for prefix in FORBIDDEN_TARGET_PREFIXES + ): + continue + # A new third-party dependency would require feature-specific flags and headers outside the + # server profile. Keep tests whose vendor dependencies are already part of the server. + if any( + dependency.startswith("third_party/") and dependency not in production_targets + for dependency in dependency_closure + ): + continue + + test_sources = _normalized_sources(target["sources"]) + source_directories = { + PurePosixPath(source).parent.as_posix() for source in test_sources + } + related = bool(related_closure & production_targets) or bool( + source_directories & production_directories + ) + if not related: + continue + if any( + source.startswith(prefix) + for source in test_sources + for prefix in EXCLUDED_TEST_TARGET_PREFIXES + ): + continue + + test_support = dependency_closure - production_targets - {main_target} + if any(targets[item]["kind"] != "Library" for item in test_support): + continue + per_test_support_sources = _normalized_sources( + source for item in test_support for source in targets[item]["sources"] + ) - production_sources + per_test_support_sources.update( + source for item in dependency_closure for source in targets[item]["sources"] + if source in initializer_sources + ) + if name == "mongo/db/pipeline/document_source_facet_test": + # The facet parser tests use this internal stage to express conflicting + # host requirements. It is a test fixture, not a server registration. + per_test_support_sources.add( + "mongo/db/pipeline/document_source_internal_split_pipeline.cpp") + if any( + source.startswith(prefix) + for source in per_test_support_sources + for prefix in EXCLUDED_TEST_TARGET_PREFIXES + ): + continue + support_targets.update(test_support) + tests.append( + { + "id": hashlib.sha1(name.encode("utf-8")).hexdigest()[:12], + "kind": "unit" if target["kind"] == "CppUnitTest" else "integration", + "name": name, + "requires_eloq": name.startswith("mongo/db/modules/eloq/") + or any(item.startswith("mongo/db/modules/eloq/") for item in dependency_closure), + "sources": sorted(test_sources), + "support_sources": sorted(per_test_support_sources), + } + ) + + support_sources = _normalized_sources( + source for name in support_targets for source in targets[name]["sources"] + ) - production_sources + support_sources.update(source for test in tests for source in test["support_sources"]) + return tests, support_sources + + +def _cmake_list(name: str, values: Iterable[str]) -> str: + body = "\n".join(f' "{value}"' for value in sorted(set(values))) + return f"set({name}\n{body}\n)\n" + + +def _cmake_scalar(name: str, value: str) -> str: + escaped = value.replace("\\", "\\\\").replace('"', '\\"') + return f'set({name} "{escaped}")\n' + + +def generate(source_root: Path, target_arch: str = platform.machine()) -> str: + targets = load_targets(source_root, target_arch) + selected = select_minimal_graph(targets, source_root, target_arch) + sources = _normalized_sources( + source + for name in selected + for source in targets[name]["sources"] + ) + tests, test_support_sources = select_minimal_tests(targets, selected) + all_test_sources = test_support_sources | { + source for test in tests for source in test["sources"] + } + + missing = sorted( + source + for source in sources + if "_gen.cpp" not in source + and source + not in GENERATED_WITHOUT_SOURCE_FILE + and not (source_root / source).is_file() + ) + if missing: + raise RuntimeError("Selected source files do not exist: " + ", ".join(missing)) + + missing_test_sources = sorted( + source + for source in all_test_sources + if "_gen.cpp" not in source + and source not in GENERATED_WITHOUT_SOURCE_FILE + and not (source_root / source).is_file() + ) + if missing_test_sources: + raise RuntimeError( + "Selected test source files do not exist: " + ", ".join(missing_test_sources) + ) + + mongo_sources = [source for source in sources if source.startswith("mongo/")] + third_party_sources = [source for source in sources if source.startswith("third_party/")] + manifest = ( + "# Generated by cmake/generate_eloqdoc_sources.py; do not edit.\n" + + _cmake_list("ELOQDOC_MONGO_TARGETS", selected) + + _cmake_list("ELOQDOC_MONGO_SOURCES", mongo_sources) + + _cmake_list("ELOQDOC_THIRD_PARTY_SOURCES", third_party_sources) + + _cmake_list("ELOQDOC_MONGO_TEST_SUPPORT_SOURCES", test_support_sources) + + _cmake_list("ELOQDOC_MONGO_TEST_IDS", (test["id"] for test in tests)) + ) + for test in tests: + prefix = f'ELOQDOC_MONGO_TEST_{test["id"]}' + manifest += _cmake_scalar(prefix + "_NAME", test["name"]) + manifest += _cmake_scalar(prefix + "_KIND", test["kind"]) + manifest += _cmake_scalar( + prefix + "_REQUIRES_ELOQ", "TRUE" if test["requires_eloq"] else "FALSE" + ) + manifest += _cmake_list(prefix + "_SOURCES", test["sources"]) + manifest += _cmake_list(prefix + "_SUPPORT_SOURCES", test["support_sources"]) + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--source-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--target-arch", default=platform.machine()) + args = parser.parse_args() + args.output.parent.mkdir(parents=True, exist_ok=True) + manifest = generate(args.source_root.resolve(), args.target_arch) + if not args.output.exists() or args.output.read_text(encoding="utf-8") != manifest: + args.output.write_text(manifest, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/cmake/generate_error_codes.py b/cmake/generate_error_codes.py new file mode 100644 index 0000000000..686c558e10 --- /dev/null +++ b/cmake/generate_error_codes.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Generate MongoDB error-code files without changing its Python 2/Cheetah generator. + +MongoDB 4.0's SCons build continues to invoke src/mongo/base/generate_error_codes.py. +The CMake build uses this Python 3-only renderer for the small Cheetah directive subset present in +the checked-in error-code templates. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path +from typing import Any, Dict, Iterable, List, Sequence, Tuple + + +DIRECTIVE = re.compile(r"^\s*//#(for|if|set)\s+(.*?)\s*$") +END_DIRECTIVE = re.compile(r"^\s*//#end\s+(for|if)\s*$") +BRACED_VALUE = re.compile(r"\$\{([^}]+)\}") +VALUE = re.compile(r"\$([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)") + + +class ErrorCode: + def __init__(self, name: str, code: int, extra: str | None = None): + self.name = name + self.code = code + self.extra = extra + self.categories: List[str] = [] + + +class ErrorClass: + def __init__(self, name: str, codes: Sequence[str]): + self.name = name + self.codes = codes + + +def parse_definitions(path: Path) -> Tuple[List[ErrorCode], List[ErrorClass]]: + error_codes: List[ErrorCode] = [] + error_classes: List[ErrorClass] = [] + namespace = { + "error_code": lambda *args, **kwargs: error_codes.append(ErrorCode(*args, **kwargs)), + "error_class": lambda *args: error_classes.append(ErrorClass(*args)), + } + source = path.read_text(encoding="utf-8") + exec(compile(source, str(path), "exec"), namespace) + error_codes.sort(key=lambda item: item.code) + + names: Dict[str, ErrorCode] = {} + numeric_codes: Dict[int, ErrorCode] = {} + for error_code in error_codes: + if error_code.name in names: + raise ValueError(f"Duplicate error-code name: {error_code.name}") + if error_code.code in numeric_codes: + raise ValueError(f"Duplicate numeric error code: {error_code.code}") + names[error_code.name] = error_code + numeric_codes[error_code.code] = error_code + + category_names = set() + for category in error_classes: + if category.name in category_names: + raise ValueError(f"Duplicate error category: {category.name}") + category_names.add(category.name) + for error_name in category.codes: + if error_name not in names: + raise ValueError( + f"Undeclared error code {error_name} in category {category.name}" + ) + names[error_name].categories.append(category.name) + return error_codes, error_classes + + +def evaluate(expression: str, values: Dict[str, Any]) -> Any: + expression = re.sub(r"\$([A-Za-z_]\w*)", r"\1", expression).rstrip(":") + scope: Dict[str, Any] = {"__builtins__": {}, "enumerate": enumerate, "len": len} + scope.update(values) + return eval(expression, scope) + + +def parse_template( + lines: Sequence[str], position: int = 0, terminator: str | None = None +) -> Tuple[List[tuple], int]: + nodes: List[tuple] = [] + while position < len(lines): + end_match = END_DIRECTIVE.match(lines[position]) + if end_match: + if end_match.group(1) != terminator: + raise ValueError(f"Unexpected template directive: {lines[position].strip()}") + return nodes, position + 1 + + match = DIRECTIVE.match(lines[position]) + if not match: + nodes.append(("text", lines[position])) + position += 1 + continue + + operation, argument = match.groups() + if operation == "for": + target, expression = argument.split(" in ", 1) + names = [name.strip().lstrip("$") for name in target.split(",")] + body, position = parse_template(lines, position + 1, "for") + nodes.append(("for", names, expression, body)) + elif operation == "if": + body, position = parse_template(lines, position + 1, "if") + nodes.append(("if", argument, body)) + else: + name, expression = argument.split("=", 1) + nodes.append(("set", name.strip().lstrip("$"), expression.strip())) + position += 1 + + if terminator: + raise ValueError(f"Missing //#end {terminator}") + return nodes, position + + +def substitute(line: str, values: Dict[str, Any]) -> str: + def replace(match: re.Match[str]) -> str: + return str(evaluate(match.group(1), values)) + + return VALUE.sub(replace, BRACED_VALUE.sub(replace, line)) + + +def render_nodes(nodes: Iterable[tuple], values: Dict[str, Any]) -> List[str]: + result: List[str] = [] + for node in nodes: + if node[0] == "text": + result.append(substitute(node[1], values)) + elif node[0] == "set": + values[node[1]] = evaluate(node[2], values) + elif node[0] == "if": + if evaluate(node[1], values): + result.extend(render_nodes(node[2], dict(values))) + else: + _, names, expression, body = node + for item in evaluate(expression, values): + nested = dict(values) + unpacked = item if len(names) > 1 else (item,) + nested.update(zip(names, unpacked)) + result.extend(render_nodes(body, nested)) + return result + + +def render_template(path: Path, values: Dict[str, Any]) -> str: + nodes, _ = parse_template(path.read_text(encoding="utf-8").splitlines(keepends=True)) + return "".join(render_nodes(nodes, values)) + + +def main(argv: Sequence[str]) -> int: + if len(argv) < 3: + raise SystemExit( + "usage: generate_error_codes.py error_codes.err template=output [template=output ...]" + ) + error_codes, error_classes = parse_definitions(Path(argv[1])) + values = {"codes": error_codes, "categories": error_classes} + for argument in argv[2:]: + template, output = argument.split("=", 1) + Path(output).write_text(render_template(Path(template), values), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/cmake/generate_icu_init.py b/cmake/generate_icu_init.py new file mode 100644 index 0000000000..58d439899e --- /dev/null +++ b/cmake/generate_icu_init.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Run SCons' unchanged ICU-data generator with Python 3 byte/text semantics.""" + +import builtins +import runpy +import sys + + +def generator_open(path, mode="r", *args, **kwargs): + # Keep the .dat input binary, but Python 2's generated C++ output is a str. + if mode == "wb": + return builtins.open(path, "w", encoding="utf-8", newline="\n") + return builtins.open(path, mode, *args, **kwargs) + + +def byte_value(value): + return value if isinstance(value, int) else ord(value) + + +def main(): + if len(sys.argv) < 2: + raise SystemExit("usage: generate_icu_init.py SCRIPT -i DATA -o OUTPUT") + sys.argv = sys.argv[1:] + runpy.run_path(sys.argv[0], run_name="__main__", + init_globals={"open": generator_open, "ord": byte_value}) + + +if __name__ == "__main__": + main() diff --git a/cmake/generate_idl.py b/cmake/generate_idl.py new file mode 100644 index 0000000000..2973382ae5 --- /dev/null +++ b/cmake/generate_idl.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Run MongoDB's Python 2 IDL compiler from the CMake/Python 3 build. + +The historical SCons build invokes scripts/buildscripts/idl/idlc.py with +Python 2. Keep that implementation untouched and provide the two small +Python 3 compatibility shims here so the build systems do not affect one +another. +""" + +import argparse +import builtins +import itertools +import sys +from pathlib import Path + + +def escape_depfile_path(path): + """Escape a path for Ninja/CMake's Make-style depfile parser.""" + return (str(path).replace("\\", "\\\\").replace("$", "$$") + .replace("#", "\\#").replace(" ", "\\ ").replace(":", "\\:")) + + +def enable_depfile(compiler, depfile): + """Record the imports actually opened by the unchanged SCons IDL compiler.""" + dependencies = set() + original_compile = compiler.compile_idl + original_open = compiler.CompilerImportResolver.open + + def tracking_open(self, resolved_file_name): + dependencies.add(Path(resolved_file_name).resolve()) + return original_open(self, resolved_file_name) + + def compile_with_dependencies(args): + dependencies.clear() + dependencies.add(Path(args.input_file).resolve()) + if not original_compile(args): + return False + source = args.output_source or str(Path(args.input_file).with_suffix("")) + "_gen.cpp" + depfile.write_text( + escape_depfile_path(Path(source).resolve()) + ": " + + " ".join(escape_depfile_path(path) for path in sorted(dependencies)) + "\n", + encoding="utf-8") + return True + + compiler.CompilerImportResolver.open = tracking_open + compiler.compile_idl = compile_with_dependencies + + +def main(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--depfile", type=Path) + args, compiler_args = parser.parse_known_args() + sys.argv[1:] = compiler_args + + idl_root = Path(__file__).resolve().parents[1] / "scripts" / "buildscripts" / "idl" + sys.path.insert(0, str(idl_root)) + + # binder.py uses Python 2's xrange. + builtins.xrange = range + + import idlc # pylint: disable=import-error,import-outside-toplevel + from idl import compiler, syntax # pylint: disable=import-error,import-outside-toplevel + + # syntax.py uses dict.viewitems(), which has no Python 3 equivalent. + def item_and_type(values): + return itertools.chain.from_iterable( + syntax._zip_scalar(value, key) for key, value in values.items()) + + syntax._item_and_type = item_and_type + if args.depfile: + enable_depfile(compiler, args.depfile) + idlc.main() + + +if __name__ == "__main__": + main() diff --git a/cmake/generate_js.py b/cmake/generate_js.py new file mode 100644 index 0000000000..469f903bf3 --- /dev/null +++ b/cmake/generate_js.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Execute the unchanged SCons JavaScript embedding generator using Python 3.""" + +import sys +from pathlib import Path + + +def main(): + script = Path(sys.argv[1]) + source = script.read_text(encoding="utf-8") + replacements = { + 'print "Must specify [target] [source] "': + 'print("Must specify [target] [source] ")', + "out.write(text)": "out.write(text.encode('utf-8'))", + } + for old, new in replacements.items(): + if source.count(old) != 1: + raise RuntimeError("unexpected jstoh.py format: " + old) + source = source.replace(old, new) + sys.argv = sys.argv[1:] + exec(compile(source, str(script), "exec"), {"__file__": str(script), "__name__": "__main__"}) + + +if __name__ == "__main__": + main() diff --git a/cmake/run_legacy_text_generator.py b/cmake/run_legacy_text_generator.py new file mode 100644 index 0000000000..04a39ebb14 --- /dev/null +++ b/cmake/run_legacy_text_generator.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Run simple Python 2 text generators under Python 3. + +These generators use binary output modes for strings or the removed ``U`` +mode. CMake needs Python 3, while SCons must keep executing the original +files with Python 2 and their historical I/O behavior. +""" + +import builtins +import runpy +import sys +from pathlib import Path + + +def main(): + if len(sys.argv) < 2: + raise SystemExit("usage: run_legacy_text_generator.py SCRIPT [ARG ...]") + + script = sys.argv[1] + sys.argv = sys.argv[1:] + sys.path.insert(0, str(Path(script).resolve().parent)) + original_open = builtins.open + + def text_open(path, mode="r", *args, **kwargs): + mode = mode.replace("b", "").replace("U", "") or "r" + return original_open(path, mode, *args, **kwargs) + + builtins.open = text_open + runpy.run_path(script, run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/cmake/tests/server_fixture.py b/cmake/tests/server_fixture.py new file mode 100644 index 0000000000..f582d07a51 --- /dev/null +++ b/cmake/tests/server_fixture.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Run a command against a private EloqDoc server; require clean server shutdown. + +The command may use {port}, {connection_string}, and {fixture} placeholders. +Unlike resmoke's external-server mode, this wrapper owns and checks the server. +Requires PyMongo in this interpreter, not in the child command's environment. +""" + +import argparse +import json +import os +from pathlib import Path +import signal +import socket +import subprocess +import tempfile +import time + +from server_smoke_config import substrate_config +from server_smoke_diagnostics import describe_exit, report_failure, wait_for_clean_shutdown + + +def reserve_ports(port): + # Substrate also uses ports adjacent to tx_port. Hold all reservations until + # immediately before launch, and fail rather than connect to someone else's server. + sockets = [] + try: + listener = socket.socket() + sockets.append(listener) + # Match the server listener: permit reuse after the preceding fixture's + # connections enter TIME_WAIT, but do not use SO_REUSEPORT/shared listeners. + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", port)) + port = listener.getsockname()[1] + for _ in range(100): + group = [] + try: + for offset in range(10): + sock = socket.socket() + group.append(sock) + sock.bind(("127.0.0.1", 0 if offset == 0 else tx_port + offset)) + if offset == 0: + tx_port = sock.getsockname()[1] + if tx_port > 65525: + raise OSError("not enough adjacent ports") + sockets.extend(group) + break + except OSError: + for sock in group: + sock.close() + else: + raise RuntimeError("Could not reserve Data Substrate ports") + host_manager = socket.socket() + sockets.append(host_manager) + host_manager.bind(("127.0.0.1", 0)) + return sockets, port, tx_port, host_manager.getsockname()[1] + except BaseException: + for sock in sockets: + sock.close() + raise + + +def child_environment(environment): + result = dict(environment) + # The wrapper uses Python 3 PyMongo. Never inject it into Python 2 resmoke + # or TPCC's independently provisioned Python 3 virtual environment. + result.pop("PYTHONPATH", None) + return result + + +def expand_command(command, port, root): + replacements = {"{port}": str(port), "{connection_string}": f"127.0.0.1:{port}", + "{fixture}": str(root)} + for old, new in replacements.items(): + command = [argument.replace(old, new) for argument in command] + return command + + +def stop_process_group(process): + # Only kill groups we created. Even when their leader exited, test clients + # and the forked host manager must not leak into the next CI phase. + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + return + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + + +def run_command(command, root, timeout): + # Retain complete output and print it live for CI's heartbeat and test summaries. + # Using a file avoids a blocked pipe if a test or one of its descendants hangs. + with (root / "tests.log").open("wb") as output: + process = subprocess.Popen(command, stdout=output, stderr=subprocess.STDOUT, + env=child_environment(os.environ), start_new_session=True) + try: + started = time.monotonic() + with (root / "tests.log").open(errors="replace") as stream: + while process.poll() is None: + print(stream.read(), end="", flush=True) + if time.monotonic() - started > timeout: + raise TimeoutError(f"Test command timed out after {timeout}s") + time.sleep(0.5) + print(stream.read(), end="", flush=True) + if process.returncode: + raise RuntimeError("Test command failed: " + describe_exit(process.returncode)) + finally: + stop_process_group(process) + + +def run_fixture(args): + from pymongo import MongoClient + from pymongo.errors import AutoReconnect, PyMongoError + + root = Path(tempfile.mkdtemp(prefix=f"eloqdoc-{args.label}-")) + print(f"Fixture: {root}", flush=True) + (root / "db").mkdir() + reservations, port, tx_port, hm_port = reserve_ports(args.port) + try: + (root / "substrate.cnf").write_text(substrate_config( + root, tx_port, hm_port, args.data_store, args.log_state, os.environ, + memory_limit_mb=args.memory_limit_mb), encoding="utf-8") + (root / "server.json").write_text(json.dumps({ + "storage": {"engine": "eloq", "dbPath": str(root / "db")}, + "net": {"port": port, "bindIp": "127.0.0.1", "serviceExecutor": "adaptive", + "adaptiveThreadNum": 1}, + "setParameter": {"enableTestCommands": True, "diagnosticDataCollectionEnabled": False, + "disableLogicalSessionCacheRefresh": True}}), encoding="utf-8") + finally: + for sock in reservations: + sock.close() + with (root / "server.log").open("wb") as log: + server = subprocess.Popen([ + str(args.server.resolve()), "--config=" + str(root / "server.json"), + "--data_substrate_config=" + str(root / "substrate.cnf"), "--log_dir=" + str(root)], + cwd=root, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + client = MongoClient("127.0.0.1", port, directConnection=True, + serverSelectionTimeoutMS=1000, socketTimeoutMS=10000) + try: + deadline = time.monotonic() + 90 + while True: + if server.poll() is not None: + raise RuntimeError("Server exited before readiness: " + describe_exit(server.returncode)) + try: + client.admin.command("ping") + break + except PyMongoError: + if time.monotonic() >= deadline: + raise TimeoutError("Server readiness timed out") + time.sleep(0.5) + run_command(expand_command(args.command, port, root), root, args.timeout) + # A successful client/test command must not hide a server crash. + if server.poll() is not None: + raise RuntimeError("Server exited during tests: " + describe_exit(server.returncode)) + try: + client.admin.command("shutdown", force=True) + except AutoReconnect: + pass # Disconnect is expected; the process exit code is authoritative. + wait_for_clean_shutdown(server, timeout=60) + print(f"PASS {args.label}: test command and server shutdown", flush=True) + except BaseException as exc: + report_failure(root, server.poll(), exc, args.diagnostics_dir) + raise + finally: + client.close() + stop_process_group(server) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--server", required=True, type=Path) + parser.add_argument("--data-store", required=True) + parser.add_argument("--log-state", required=True) + parser.add_argument("--label", choices=("integration", "jstests", "tpcc"), required=True) + parser.add_argument("--port", type=int, default=27017) + parser.add_argument("--memory-limit-mb", type=int, default=4000) + parser.add_argument("--timeout", type=int, default=7200) + parser.add_argument("--diagnostics-dir", type=Path) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + if args.command[:1] == ["--"]: + args.command.pop(0) + if not args.command: + parser.error("a test command after -- is required") + run_fixture(args) + + +if __name__ == "__main__": + main() diff --git a/cmake/tests/server_smoke.py b/cmake/tests/server_smoke.py new file mode 100644 index 0000000000..c47958dbd2 --- /dev/null +++ b/cmake/tests/server_smoke.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Exercise the real CMake server over the wire (requires pymongo>=4.6,<4.11). + +Uses a private Data Substrate fixture with one or two worker cores. Supports local RocksDB +(WAL disabled) and EloqStore/S3 with RocksDB/S3 logging (WAL enabled). The S3 fixture +requires S3_ENDPOINT, S3_ACCESS_KEY and S3_SECRET_KEY and a running disposable S3 +service. Logs and local data are retained in the printed temporary directory. +""" + +import argparse +import datetime +import json +import os +from pathlib import Path +import socket +import signal +import subprocess +import tempfile +import threading +import time + +from pymongo import MongoClient +from bson.int64 import Int64 +from pymongo.errors import AutoReconnect, OperationFailure, PyMongoError +from pymongo.read_concern import ReadConcern + +from server_smoke_config import substrate_config +from server_smoke_diagnostics import describe_exit, report_failure, wait_for_clean_shutdown + + +def free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def eventually(predicate, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.25) + raise AssertionError("condition did not become true within %ss" % timeout) + + +def command_fails(database, command, *, code=None, message): + try: + database.command(command) + except OperationFailure as exc: + if code is not None: + assert exc.code == code, exc + assert message in str(exc), exc + else: + raise AssertionError("command unexpectedly succeeded: " + repr(command)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--server", required=True, type=Path) + parser.add_argument("--data-store", default="ELOQDSS_ROCKSDB") + parser.add_argument("--log-state", default="ROCKSDB") + parser.add_argument("--ttl-shutdown-mode", choices=("sleeping", "active"), default="sleeping") + parser.add_argument("--diagnostics-dir", type=Path, + help="Copy failure logs here for CI artifact collection") + args = parser.parse_args() + server = args.server.resolve() + # Keep every fixture child, including auxiliary services, within the user's CPU limit. + if hasattr(os, "sched_getaffinity"): + os.sched_setaffinity(0, sorted(os.sched_getaffinity(0))[:8]) + root = Path(tempfile.mkdtemp(prefix="eloqdoc-server-smoke-")) + print("Fixture:", root, flush=True) + (root / "db").mkdir() + port, tx_port = free_port(), free_port() + config = root / "data_substrate.cnf" + try: + config_text = substrate_config(root, tx_port, free_port(), args.data_store, + args.log_state, os.environ, + # One worker makes a blocking shutdown wait deadlock + # deterministically if TTL still needs that worker. + core_number=1 if args.ttl_shutdown_mode == "active" else 2) + except ValueError as exc: + parser.error(str(exc)) + config.write_text(config_text, encoding="utf-8") + server_config = root / "eloqdoc.yaml" + # JSON is valid YAML and avoids adding another test-client dependency. + server_config.write_text(json.dumps({ + "storage": {"dbPath": str(root / "db")}, + "net": {"port": port, "bindIp": "127.0.0.1", + "serviceExecutor": "adaptive", "adaptiveThreadNum": 1}, + }), encoding="utf-8") + common = [str(server), "--data_substrate_config=" + str(config), + "--config=" + str(server_config), "--log_dir=" + str(root)] + for option in ("--replSet=test", "--shardsvr", "--configsvr", "--storageEngine=wiredTiger"): + result = subprocess.run(common + [option], cwd=root, capture_output=True, timeout=30) + (root / (option.split("=")[0][2:] + ".log")).write_bytes(result.stdout + result.stderr) + assert result.returncode > 0, option + " was accepted or crashed instead of being rejected" + expected = (b"unknown storage engine: wiredTiger" if "storageEngine" in option + else b"supports standalone mode only") + assert expected in result.stdout + result.stderr, "unexpected failure for " + option + print("PASS unsupported topology/storage options rejected", flush=True) + + with (root / "server.log").open("wb") as log: + process = subprocess.Popen(common + ["--auth", "--setParameter=ttlMonitorSleepSecs=1", + # Exercise the two test-only maintenance-mode commands too. + "--setParameter=enableTestCommands=1", + "--setParameter=diagnosticDataCollectionEnabled=false"], + cwd=root, stdout=log, stderr=subprocess.STDOUT, + start_new_session=True) + anonymous = MongoClient("127.0.0.1", port, directConnection=True, + serverSelectionTimeoutMS=1000, socketTimeoutMS=15000) + authenticated = None + try: + last_connection_error = [] + def ready(): + returncode = process.poll() + assert returncode is None, "server exited: " + describe_exit(returncode) + try: + return anonymous.admin.command("ping")["ok"] == 1 + except PyMongoError as exc: + last_connection_error[:] = [str(exc)] + return False + try: + eventually(ready, 90) + except AssertionError as exc: + raise AssertionError(f"{exc}; last client error: {last_connection_error}") from exc + hello = anonymous.admin.command("ismaster") + assert hello["ismaster"] and "setName" not in hello + anonymous.admin.command("createUser", "smoke_root", pwd="temporary-smoke-password", + roles=["root"]) + authenticated = MongoClient("127.0.0.1", port, directConnection=True, + username="smoke_root", password="temporary-smoke-password", + authSource="admin", serverSelectionTimeoutMS=5000, + socketTimeoutMS=15000) + db = authenticated.cmake_smoke + try: + anonymous.cmake_smoke.records.insert_one({"unauthorized": True}) + raise AssertionError("unauthenticated write was permitted with --auth") + except OperationFailure as exc: + assert exc.code == 13, exc + print("PASS authentication and authorization", flush=True) + commands = authenticated.admin.command("listCommands")["commands"] + for command in ("isMaster", "find", "insert", "aggregate", "mapReduce", "createUser", + "createIndexes", "shutdown", "fsync", "compact", "touch", + "repairDatabase", "invalidateViewCatalog", "restartCatalog"): + assert command in commands, "missing standalone command: " + command + for command in ("replSetInitiate", "replSetReconfig", "_configsvrAddShard", + "mapreduce.shardedfinish", "shardConnPoolStats"): + assert command not in commands, "distributed command retained: " + command + print("PASS standalone command registration", flush=True) + db.records.insert_many([{"k": i, "v": i % 2} for i in range(10)]) + db.records.create_index("k", unique=True, background=True) + assert db.records.find_one({"k": 3})["v"] == 1 + db.records.update_one({"k": 3}, {"$set": {"v": 7}}) + assert db.records.find_one({"k": 3})["v"] == 7 + rows = list(db.records.aggregate([{"$group": {"_id": None, "n": {"$sum": 1}}}])) + assert rows[0]["n"] == 10 + db.records.delete_one({"k": 9}) + assert db.records.count_documents({}) == 9 + print("PASS CRUD, background index, local aggregation", flush=True) + # Collation metadata must remain compatible with the historical SCons build. + collation = {"locale": "en_US", "strength": 2, "version": "57.1"} + db.collated.create_index("s", collation=collation) + db.collated.insert_one({"s": "Apple"}) + assert db.collated.find_one({"s": "APPLE"}, collation=collation)["s"] == "Apple" + index = next(item for item in db.collated.list_indexes() if item["name"] == "s_1") + assert index["collation"]["version"] == "57.1", index + print("PASS SCons-compatible ICU 57.1 collation metadata and lookup", flush=True) + # These must reach their handlers, which report the existing engine restrictions, + # instead of throwing NoReplicationEnabled from MaintenanceModeSetter. + command_fails(db, {"compact": "records"}, code=115, message="cannot compact") + command_fails(db, {"touch": "records", "data": True}, code=115, + message="does not support touch") + command_fails(db, {"repairDatabase": 1}, + message="Eloq storage engine does not require manual database repair") + db.command("create", "records_view", viewOn="records", pipeline=[]) + db.command("invalidateViewCatalog", 1) + assert db.records_view.count_documents({}) == 9 + print("PASS maintenance-mode handlers and view catalog reload", flush=True) + # getMore with a term requires internal authorization even on a standalone server. + authenticated.admin.command("createUser", "smoke_internal", + pwd="temporary-internal-password", roles=["__system"]) + with MongoClient("127.0.0.1", port, directConnection=True, + username="smoke_internal", password="temporary-internal-password", + authSource="admin", serverSelectionTimeoutMS=5000, + socketTimeoutMS=15000) as internal: + cursor = internal.cmake_smoke.command("find", "records", batchSize=1)["cursor"] + assert cursor["id"] != 0 + try: + command_fails(internal.cmake_smoke, + {"getMore": cursor["id"], "collection": "records", "term": Int64(1)}, + code=2, message="cannot supply 'term' without active replication") + finally: + internal.cmake_smoke.command("killCursors", "records", cursors=[cursor["id"]]) + authenticated.admin.command("dropUser", "smoke_internal") + assert authenticated.admin.command("getParameter", 1, replIndexPrefetch=1)[ + "replIndexPrefetch"] == "uninitialized" + command_fails(authenticated.admin, {"setParameter": 1, "replIndexPrefetch": "none"}, + code=2, message="replication is not enabled") + print("PASS getMore term and replication parameter status codes", flush=True) + assert len(list(db.records.find({"$where": "function() { return this.k === 2; }"}))) == 1 + print("PASS server-side JavaScript", flush=True) + # The existing Eloq command explicitly disallows map-reduce, independently of JS. + try: + db.command("mapReduce", "records", + map="function() { emit(this.v, 1); }", + reduce="function(key, values) { return Array.sum(values); }", + out={"inline": 1}) + raise AssertionError("existing Eloq map-reduce restriction changed") + except OperationFailure as exc: + assert "Eloq storage engine does not support mapreduce" in str(exc), exc + print("PASS existing Eloq map-reduce restriction preserved", flush=True) + db.expiring.create_index("expires", expireAfterSeconds=0) + db.expiring.insert_one({"expires": datetime.datetime.now(datetime.timezone.utc) + - datetime.timedelta(minutes=5)}) + eventually(lambda: db.expiring.count_documents({}) == 0, 30) + print("PASS TTL deletion by background worker", flush=True) + with authenticated.start_session() as session: + assert db.command("ping", session=session)["ok"] == 1 + with session.start_transaction(read_concern=ReadConcern("local")): + db.records.update_one({"k": 0}, {"$set": {"v": 20}}, session=session) + db.records.update_one({"k": 1}, {"$set": {"v": 21}}, session=session) + assert db.records.find_one({"k": 0})["v"] == 20 + assert db.records.find_one({"k": 1})["v"] == 21 + session.start_transaction(read_concern=ReadConcern("local")) + db.records.update_one({"k": 0}, {"$set": {"v": -1}}, session=session) + session.abort_transaction() + assert db.records.find_one({"k": 0})["v"] == 20 + # Check the normal background refresh, without invoking refreshLogicalSessionCacheNow. + eventually(lambda: "lsidTTLIndex" in + authenticated.config.system.sessions.index_information(), 30) + print("PASS logical sessions, transaction commit/abort, and maintenance", flush=True) + authenticated.admin.command("fsync") + authenticated.admin.command({"fsync": 1, "async": True}) + # Exercise the lock thread's flush and backup path as well. Eloq's KV engine does + # not implement backup mode, so the normal storage-engine error must survive. + command_fails(authenticated.admin, {"fsync": 1, "lock": True}, code=115, + message="doesn't support backup mode") + status = authenticated.admin.command("serverStatus") + assert status["storageEngine"]["name"] == "eloq" + assert "setName" not in status.get("repl", {}) + print("PASS fsync and server diagnostics", flush=True) + for _ in range(2): + # Eloq's no-op global locker cannot protect a live catalog reload across workers. + # Reject safely before invalidating any catalog, then verify the server still works. + command_fails(authenticated.admin, {"restartCatalog": 1}, code=115, + message="does not support online catalog restart") + assert db.records.count_documents({}) == 9 + assert db.records_view.count_documents({}) == 9 + assert db.records.find_one({"k": 0})["v"] == 20 + print("PASS unsafe online catalog restart rejected without affecting data", flush=True) + def server_logged(message): + return message in (root / "server.log").read_text(encoding="utf-8", errors="replace") + if args.ttl_shutdown_mode == "sleeping": + # Confirm the worker enters a wait much longer than the shutdown timeout. + # Merely joining an uninterruptible sleep must fail this regression check. + authenticated.admin.command({"setParameter": 1, + "logComponentVerbosity": {"index": {"verbosity": 1}}, + "ttlMonitorSleepSecs": 600}) + eventually(lambda: server_logged("TTL monitor waiting for next pass: 600 seconds"), 10) + print("PASS TTL worker entered its long sleep before shutdown", flush=True) + else: + authenticated.admin.command({"configureFailPoint": "hangTTLMonitorBeforeStorageAccess", + "mode": "alwaysOn"}) + eventually(lambda: server_logged("TTL monitor paused before storage access"), 10) + print("PASS active TTL pass held before shutdown", flush=True) + # The network integration suites can leave a long-running command behind. Shutdown + # must interrupt it, not wait for its normal completion or require SIGTERM. + sleep_finished = threading.Event() + sleep_errors = [] + def sleeping_command(): + try: + authenticated.admin.command({"sleep": 1, "secs": 600, "lock": "none", + "comment": "shutdown-smoke"}) + sleep_errors.append("sleep completed before shutdown") + except AutoReconnect: + pass + except OperationFailure as exc: + if exc.code not in (91, 11600): # ShutdownInProgress, InterruptedAtShutdown + sleep_errors.append(str(exc)) + except Exception as exc: + sleep_errors.append(str(exc)) + finally: + sleep_finished.set() + sleeper = threading.Thread(target=sleeping_command, daemon=True) + sleeper.start() + def sleep_is_active(): + assert not sleep_finished.is_set(), sleep_errors + return any(op.get("command", {}).get("comment") == "shutdown-smoke" + for op in authenticated.admin.command("currentOp")["inprog"]) + eventually(sleep_is_active, 10) + try: + authenticated.admin.command("shutdown", force=True) + except AutoReconnect: + pass + wait_for_clean_shutdown(process, timeout=30) + assert sleep_finished.wait(5), "sleep command did not finish during shutdown" + assert not sleep_errors, sleep_errors + shutdown_log = (root / "server.log").read_text(encoding="utf-8", errors="replace") + ttl_stopped = shutdown_log.index("TTL monitor stopped") + storage_stopped = shutdown_log.index("EloqKVEngine::cleanShutdown") + assert ttl_stopped < storage_stopped, "storage teardown ran before TTL completed" + if args.ttl_shutdown_mode == "active": + assert shutdown_log.index("TTL monitor resuming storage access") < ttl_stopped + print("PASS TTL worker drained before storage teardown", flush=True) + print("PASS clean shutdown with an active command", flush=True) + except BaseException as exc: + # Record the real exit status and logs before fixture cleanup can send signals. + report_failure(root, process.poll(), exc, args.diagnostics_dir) + raise + finally: + anonymous.close() + if authenticated: + authenticated.close() + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + # The fixture owns this new session. Also stop auxiliary children after a crash. + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + + +if __name__ == "__main__": + main() diff --git a/cmake/tests/server_smoke_config.py b/cmake/tests/server_smoke_config.py new file mode 100644 index 0000000000..49b63cffa6 --- /dev/null +++ b/cmake/tests/server_smoke_config.py @@ -0,0 +1,55 @@ +"""Data Substrate configuration for the Mongo-facing server smoke fixture.""" + +import uuid + + +def substrate_config(root, tx_port, hm_port, data_store, log_state, environment, *, core_number=2, + memory_limit_mb=None): + cloud = (data_store, log_state) == ("ELOQDSS_ELOQSTORE", "ROCKSDB_CLOUD_S3") + local = data_store in ("ROCKSDB", "ELOQDSS_ROCKSDB") and log_state == "ROCKSDB" + if not (cloud or local): + raise ValueError(f"Unsupported smoke-test backend: {data_store}/{log_state}") + if memory_limit_mb is None: + memory_limit_mb = 2048 if cloud else 512 + + cloud_local, cloud_store = "", "" + if cloud: + required = ("S3_ENDPOINT", "S3_ACCESS_KEY", "S3_SECRET_KEY") + missing = [key for key in required if not environment.get(key)] + if missing: + raise ValueError("EloqStore/S3 smoke test requires " + ", ".join(missing)) + endpoint = environment["S3_ENDPOINT"].rstrip("/") + access_key, secret_key = environment["S3_ACCESS_KEY"], environment["S3_SECRET_KEY"] + # A fresh bucket isolates each invocation, including retries. CI owns the disposable + # S3 service; the smoke test never removes buckets from an externally supplied service. + bucket = "eloqdoc-smoke-" + uuid.uuid4().hex + cloud_local = ( + "checkpoint_interval=10\nlogserver_snapshot_interval=60\n" + "txlog_rocksdb_cloud_bucket_prefix=\n" + f"txlog_rocksdb_cloud_bucket_name={bucket}\n" + "txlog_rocksdb_cloud_object_path=txlog\n" + f"txlog_rocksdb_cloud_s3_endpoint_url={endpoint}\n" + f"txlog_rocksdb_cloud_object_store_service_url={endpoint}/{bucket}/txlog\n" + "txlog_rocksdb_cloud_sst_file_cache_size=1GB\n") + cloud_store = ( + "[store]\n" + f"aws_access_key_id={access_key}\naws_secret_key={secret_key}\n" + "eloq_store_buffer_pool_size=500MB\neloq_store_local_space_limit=2GB\n" + "eloq_store_manifest_limit=1048576\neloq_store_pages_per_file_shift=8\n" + "eloq_store_cloud_provider=aws\n" + f"eloq_store_cloud_endpoint={endpoint}\n" + f"eloq_store_cloud_store_path={bucket}/eloqstore\n" + f"eloq_store_cloud_access_key={access_key}\n" + f"eloq_store_cloud_secret_key={secret_key}\n" + "eloq_store_cloud_verify_ssl=false\neloq_store_cloud_request_threads=2\n") + + return ( + f"[local]\ncore_number={core_number}\n" + f"node_memory_limit_mb={memory_limit_mb}\n" + f"enable_data_store=true\nenable_wal={'true' if cloud else 'false'}\n" + "enable_mvcc=true\nevent_dispatcher_num=1\nbind_all=false\n" + f"tx_ip=127.0.0.1\ntx_port={tx_port}\nhm_port={hm_port}\n" + f"eloq_data_path={root / 'eloq'}\nlog_service_data_path={root / 'log'}\n" + + cloud_local + + f"[cluster]\ntx_ip_port_list=127.0.0.1:{tx_port}\n" + + cloud_store) diff --git a/cmake/tests/server_smoke_diagnostics.py b/cmake/tests/server_smoke_diagnostics.py new file mode 100644 index 0000000000..2e239e2f8a --- /dev/null +++ b/cmake/tests/server_smoke_diagnostics.py @@ -0,0 +1,72 @@ +"""Failure diagnostics for the Mongo-facing smoke test (standard library only).""" + +import shutil +import signal +import subprocess +import sys + + +def describe_exit(returncode): + if returncode is None: + return "still running" + description = f"exit code {returncode}" + if returncode < 0: + try: + description += f" ({signal.Signals(-returncode).name})" + except ValueError: + description += f" (signal {-returncode})" + return description + + +def wait_for_clean_shutdown(process, timeout=30): + try: + returncode = process.wait(timeout=timeout) + except subprocess.TimeoutExpired as exc: + raise AssertionError(f"server shutdown timed out after {timeout}s") from exc + if returncode != 0: + raise AssertionError("server shutdown failed: " + describe_exit(returncode)) + + +def report_failure(root, returncode, error, diagnostics_dir=None): + """Print bounded log tails and preserve full logs, without masking the original failure.""" + summary = (f"Fixture: {root}\n" + f"Failure: {type(error).__name__}: {error}\n" + f"Server status before fixture cleanup: {describe_exit(returncode)}\n") + print(summary, file=sys.stderr, flush=True) + destination = None + if diagnostics_dir is not None: + try: + destination = diagnostics_dir / root.name + destination.mkdir(parents=True, exist_ok=True) + (destination / "failure.txt").write_text(summary, encoding="utf-8") + except OSError as exc: + print(f"Could not save failure summary: {exc}", file=sys.stderr, flush=True) + destination = None + + try: + # Include Mongo output, rejected-option logs, and timestamped glog files. Do not + # upload data directories, core dumps, credential-bearing configs, or symlinks. + logs = sorted(path for path in root.iterdir() + if not path.is_symlink() and path.is_file() + and (path.name.endswith(".log") or ".log." in path.name)) + except OSError as exc: + print(f"Could not list fixture logs: {exc}", file=sys.stderr, flush=True) + return + for path in logs: + try: + # Limit console output, but keep the complete file in the artifact. + with path.open("rb") as log: + log.seek(0, 2) + log.seek(max(0, log.tell() - 64 * 1024)) + tail = log.read().decode("utf-8", errors="replace").splitlines()[-200:] + print(f"--- {path.name} (last up to 200 lines / 64 KiB) ---\n" + "\n".join(tail), + file=sys.stderr, flush=True) + except OSError as exc: + print(f"Could not read {path.name}: {exc}", file=sys.stderr, flush=True) + if destination is not None: + try: + shutil.copyfile(path, destination / path.name) + except OSError as exc: + print(f"Could not preserve {path.name}: {exc}", file=sys.stderr, flush=True) + if destination is not None: + print(f"Failure diagnostics: {destination}", file=sys.stderr, flush=True) diff --git a/cmake/tests/test_icu_generator.py b/cmake/tests/test_icu_generator.py new file mode 100644 index 0000000000..cc675714ac --- /dev/null +++ b/cmake/tests/test_icu_generator.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Check the Python 3 adapter preserves the original ICU data and initializer.""" + +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest + + +class ICUGeneratorTest(unittest.TestCase): + def test_all_byte_values_are_embedded_unchanged(self): + root = Path(__file__).resolve().parents[2] + with tempfile.TemporaryDirectory() as directory: + data = Path(directory) / "data.dat" + output = Path(directory) / "icu_init.cpp" + payload = bytes(range(256)) + b"\r\n\x00\xff" + data.write_bytes(payload) + subprocess.run([ + sys.executable, str(root / "cmake/generate_icu_init.py"), + str(root / "src/mongo/util/generate_icu_init_cpp.py"), + "-i", str(data), "-o", str(output)], check=True) + source = output.read_text() + encoded = re.search(r"kRawData\[\] = \{([\d,]+)\};", source) + self.assertIsNotNone(encoded) + self.assertEqual(payload, bytes(int(value) for value in encoded.group(1).split(","))) + self.assertIn("alignas(16)", source) + self.assertIn("MONGO_INITIALIZER(LoadICUData)", source) + self.assertIn("udata_setCommonData(kRawData, &status)", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/tests/test_idl_dependencies.py b/cmake/tests/test_idl_dependencies.py new file mode 100644 index 0000000000..882c4eea44 --- /dev/null +++ b/cmake/tests/test_idl_dependencies.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Exercise the real IDL compiler and Ninja's transitive dependency tracking.""" + +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from generate_idl import escape_depfile_path + + +class IDLDependenciesTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="eloqdoc idl ") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.build = self.root / "build" + self.wrapper = Path(__file__).resolve().parents[1] / "generate_idl.py" + leaf = '''types: + testString: + description: A test string + cpp_type: std::string + bson_serialization_type: string + deserializer: mongo::BSONElement::str +''' + (self.root / "leaf.idl").write_text(leaf, encoding="utf-8") + (self.root / "other.idl").write_text(leaf, encoding="utf-8") + (self.root / "middle.idl").write_text("imports: [leaf.idl]\n", encoding="utf-8") + for name, imported in (("first", "middle.idl"), ("second", "other.idl")): + (self.root / (name + ".idl")).write_text( + "imports: [" + imported + "]\n" + "global:\n cpp_namespace: mongo\n" + "structs:\n Example:\n description: Test struct\n" + " fields:\n value: testString\n", encoding="utf-8") + (self.root / "CMakeLists.txt").write_text(''' +cmake_minimum_required(VERSION 3.24) +project(IDLDependencies NONE) +foreach(name IN ITEMS first second) + set(output "${CMAKE_CURRENT_BINARY_DIR}/${name}_gen.cpp") + add_custom_command( + OUTPUT "${output}" "${CMAKE_CURRENT_BINARY_DIR}/${name}_gen.h" + COMMAND "${IDL_PYTHON}" "${IDL_WRAPPER}" + --include "${CMAKE_CURRENT_SOURCE_DIR}" + --base_dir "${CMAKE_CURRENT_BINARY_DIR}" + --output "${output}" + --header "${CMAKE_CURRENT_BINARY_DIR}/${name}_gen.h" + --depfile "${output}.d" + "${CMAKE_CURRENT_SOURCE_DIR}/${name}.idl" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${name}.idl" "${IDL_WRAPPER}" + DEPFILE "${output}.d" + VERBATIM) + add_custom_target(${name} ALL DEPENDS "${output}") +endforeach() +''', encoding="utf-8") + self.run_command("cmake", "-S", self.root, "-B", self.build, "-G", "Ninja", + "-DIDL_PYTHON=" + sys.executable, "-DIDL_WRAPPER=" + str(self.wrapper)) + + def run_command(self, *args): + result = subprocess.run([str(arg) for arg in args], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return result.stdout + + def build_outputs(self): + self.run_command("cmake", "--build", self.build, "-j2") + return tuple((self.build / (name + "_gen.cpp")).stat().st_mtime_ns + for name in ("first", "second")) + + def test_depfile_lists_only_actual_transitive_inputs(self): + self.build_outputs() + depfile = (self.build / "first_gen.cpp.d").read_text(encoding="utf-8") + for name in ("first.idl", "middle.idl", "leaf.idl"): + self.assertIn(escape_depfile_path(self.root / name), depfile) + self.assertNotIn("other.idl", depfile) + self.assertNotIn("second.idl", depfile) + + def test_incremental_build_tracks_changed_and_new_imports(self): + original = self.build_outputs() + self.assertEqual(original, self.build_outputs(), "no-op build regenerated IDL") + with (self.root / "leaf.idl").open("a", encoding="utf-8") as stream: + stream.write("\n# Changed transitive dependency\n") + changed = self.build_outputs() + self.assertNotEqual(original[0], changed[0]) + self.assertEqual(original[1], changed[1], "unrelated output was regenerated") + + (self.root / "new.idl").write_text("global:\n cpp_namespace: mongo\n", + encoding="utf-8") + (self.root / "middle.idl").write_text("imports: [leaf.idl, new.idl]\n", + encoding="utf-8") + imported = self.build_outputs() + self.assertNotEqual(changed[0], imported[0]) + self.assertEqual(changed[1], imported[1]) + with (self.root / "new.idl").open("a", encoding="utf-8") as stream: + stream.write("\n# Newly discovered dependency changed\n") + updated = self.build_outputs() + self.assertNotEqual(imported[0], updated[0]) + self.assertEqual(imported[1], updated[1]) + self.assertEqual(updated, self.build_outputs()) + + def test_failed_compilation_does_not_publish_depfile(self): + (self.root / "first.idl").write_text("imports: [missing.idl]\n", encoding="utf-8") + result = subprocess.run(["cmake", "--build", str(self.build), "--target", "first", "-j2"], + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertFalse((self.build / "first_gen.cpp.d").exists()) + + def test_depfile_does_not_change_generated_cpp_or_header(self): + self.build_outputs() + cpp = self.build / "first_gen.cpp" + header = self.build / "first_gen.h" + expected = (cpp.read_bytes(), header.read_bytes()) + self.run_command(sys.executable, self.wrapper, "--include", self.root, + "--base_dir", self.build, "--output", cpp, "--header", header, + self.root / "first.idl") + self.assertEqual(expected, (cpp.read_bytes(), header.read_bytes())) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/tests/test_scons_test_selection.py b/cmake/tests/test_scons_test_selection.py new file mode 100644 index 0000000000..a5692b118c --- /dev/null +++ b/cmake/tests/test_scons_test_selection.py @@ -0,0 +1,63 @@ +"""Tests for the explicit SCons EloqDoc validation profile (Python 3).""" + +import importlib.util +from pathlib import Path +import unittest + +spec = importlib.util.spec_from_file_location( + "select_eloq_unit_tests", Path(__file__).resolve().parents[2] / + "scripts/buildscripts/select_eloq_unit_tests.py") +selector = importlib.util.module_from_spec(spec) +spec.loader.exec_module(selector) + + +class EloqUnitTestSelectionTest(unittest.TestCase): + def test_unsupported_engines_and_embedded_are_skipped(self): + for target in ("mongo/db/storage/mmap_v1/btree/btree_interface_test", + "mongo/db/storage/wiredtiger/storage_wiredtiger_prefixed_record_store_test", + "mongo/embedded/mongo_embedded_capi_test"): + for prefix in ("build/gcc15/", "/tmp/test-build/gcc15/"): + self.assertIsNotNone(selector.skip_reason(prefix + target)) + + def test_obsolete_interfaces_have_narrow_exclusions(self): + self.assertIsNotNone(selector.skip_reason("build/gcc15/mongo/db/views/views_test")) + self.assertIsNone(selector.skip_reason("build/gcc15/mongo/db/pipeline/pipeline_test")) + + def test_replication_suites_are_excluded_including_future_targets(self): + for name in ("replication_process_test", "replication_consistency_markers_impl_test", + "drop_pending_collection_reaper_test", "replication_coordinator_impl_test", + "initial_syncer_test", "future_test"): + for prefix in ("", "build/gcc15/", "/tmp/test-build/gcc15/"): + self.assertIn("replication", selector.skip_reason(prefix + "mongo/db/repl/" + name)) + + def test_fixed_tests_and_applicable_catalog_coverage_remain_selected(self): + for name in ("mongo/db/query/collation/collator_factory_icu_test", + "mongo/db/operation_time_tracker_test", "mongo/util/background_job_test", + "mongo/s/query/async_results_merger_test", + "mongo/db/catalog/collection_test", + "mongo/db/storage/kv/kv_collection_catalog_entry_test", + "mongo/db/storage/storage_repair_observer_test", + "mongo/db/standalone_runtime_test", + "mongo/db/free_mon/free_mon_test", + "mongo/db/storage/ephemeral_for_test/storage_ephemeral_for_test_record_store_test"): + self.assertIsNone(selector.skip_reason(name)) + + def test_confirmed_legacy_fixtures_have_explicit_reasons(self): + for target, reason in selector.UNSUPPORTED_RUNTIME_FIXTURES.items(): + self.assertTrue(reason) + self.assertEqual(reason, selector.skip_reason("/tmp/build/gcc15/" + target)) + self.assertIn("oplog", selector.skip_reason("mongo/db/catalog/database_test")) + self.assertIn("renameCollection", + selector.skip_reason("mongo/db/catalog/create_collection_test")) + self.assertIn("repair", selector.skip_reason("mongo/db/storage/kv/kv_storage_engine_test")) + # New tests in these directories must be assessed, not silently skipped. + self.assertIsNone(selector.skip_reason("mongo/db/catalog/future_test")) + self.assertIsNone(selector.skip_reason("mongo/db/s/future_test")) + + def test_missing_or_unrecognized_binary_is_not_silently_skipped(self): + binaries = ["/no-such-build/mongo/db/future_test", "unknown_test"] + self.assertEqual((binaries, []), selector.select_tests(binaries)) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/tests/test_server_fixture.py b/cmake/tests/test_server_fixture.py new file mode 100644 index 0000000000..6f3b32ef3d --- /dev/null +++ b/cmake/tests/test_server_fixture.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Regression checks for CI scope, client isolation, and runtime result propagation.""" + +import contextlib +import io +from pathlib import Path +import socket +import sys +import tempfile +import types +import unittest +from unittest import mock + +import yaml + +from server_fixture import child_environment, expand_command, reserve_ports, run_command, run_fixture + + +class ServerFixtureTest(unittest.TestCase): + def test_port_reservation_allows_time_wait_but_rejects_a_busy_port(self): + listener = mock.Mock() + listener.bind.side_effect = OSError("Address already in use") + with mock.patch("server_fixture.socket.socket", return_value=listener): + with self.assertRaisesRegex(OSError, "Address already in use"): + reserve_ports(27017) + listener.setsockopt.assert_called_once_with(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.close.assert_called_once() + + def test_client_python_environments_are_isolated(self): + environment = {"PYTHONPATH": "/python3/pymongo", "PATH": "/shell/bin:/usr/bin", + "LD_LIBRARY_PATH": "/third-party/lib"} + self.assertEqual({key: value for key, value in environment.items() if key != "PYTHONPATH"}, + child_environment(environment)) + self.assertIn("PYTHONPATH", environment) + + def test_command_arguments_are_substituted_without_shell_expansion(self): + command = ["ctest", "--connection={connection_string}", "--port={port}", + "--report={fixture}/report.json", "{not_a_placeholder}"] + self.assertEqual(["ctest", "--connection=127.0.0.1:12345", "--port=12345", + "--report=/tmp/path with spaces/report.json", "{not_a_placeholder}"], + expand_command(command, 12345, Path("/tmp/path with spaces"))) + + def test_test_failure_is_not_hidden_by_fixture_cleanup(self): + with tempfile.TemporaryDirectory() as directory, contextlib.redirect_stdout(io.StringIO()): + root = Path(directory) + with self.assertRaisesRegex(RuntimeError, "exit code 23"): + run_command([sys.executable, "-c", "print('failure context'); raise SystemExit(23)"], + root, 5) + self.assertIn("failure context", (root / "tests.log").read_text()) + + def test_hung_command_times_out(self): + with tempfile.TemporaryDirectory() as directory, contextlib.redirect_stdout(io.StringIO()): + with self.assertRaisesRegex(TimeoutError, "timed out"): + run_command([sys.executable, "-c", "import time; time.sleep(60)"], Path(directory), 1) + + def test_successful_command_is_recorded(self): + with tempfile.TemporaryDirectory() as directory, contextlib.redirect_stdout(io.StringIO()): + root = Path(directory) + run_command([sys.executable, "-c", "print('passed')"], root, 5) + self.assertEqual("passed\n", (root / "tests.log").read_text()) + + def check_server_exit(self, returncode): + # No PyMongo installation or network access needed for these lifecycle checks. + client = mock.Mock() + pymongo = types.ModuleType("pymongo") + pymongo.MongoClient = mock.Mock(return_value=client) + errors = types.ModuleType("pymongo.errors") + errors.PyMongoError = type("PyMongoError", (Exception,), {}) + errors.AutoReconnect = type("AutoReconnect", (errors.PyMongoError,), {}) + server = mock.Mock() + server.poll.side_effect = [None, None, returncode] + server.wait.return_value = returncode + with tempfile.TemporaryDirectory() as directory, contextlib.ExitStack() as stack: + stack.enter_context(contextlib.redirect_stdout(io.StringIO())) + stack.enter_context(mock.patch.dict(sys.modules, {"pymongo": pymongo, + "pymongo.errors": errors})) + stack.enter_context(mock.patch("server_fixture.tempfile.mkdtemp", return_value=directory)) + stack.enter_context(mock.patch("server_fixture.reserve_ports", + return_value=([], 27017, 16379, 16389))) + stack.enter_context(mock.patch("server_fixture.subprocess.Popen", return_value=server)) + stack.enter_context(mock.patch("server_fixture.run_command")) + cleanup = stack.enter_context(mock.patch("server_fixture.stop_process_group")) + report = stack.enter_context(mock.patch("server_fixture.report_failure")) + args = types.SimpleNamespace(label="integration", port=27017, memory_limit_mb=4000, + server=Path("/tmp/eloqdoc"), data_store="ELOQDSS_ROCKSDB", + log_state="ROCKSDB", command=["ctest"], timeout=10, + diagnostics_dir=None) + if returncode: + with self.assertRaisesRegex(AssertionError, "server shutdown failed"): + run_fixture(args) + report.assert_called_once() + else: + run_fixture(args) + report.assert_not_called() + cleanup.assert_called_once_with(server) + client.close.assert_called_once() + client.admin.command.assert_any_call("shutdown", force=True) + + def test_passing_tests_do_not_hide_a_server_crash(self): + self.check_server_exit(-11) + + def test_passing_tests_require_clean_shutdown(self): + self.check_server_exit(0) + + def test_ci_enables_all_runtime_phases(self): + source = Path(__file__).resolve().parents[2] + workflow = yaml.safe_load((source / ".github/workflows/ci.yml").read_text()) + job = workflow["jobs"]["cmake-build-smoke"] + self.assertNotIn("strategy", job) + commands = "\n".join(step.get("run", "") for step in job["steps"]) + self.assertIn("-DELOQDOC_REGISTER_INTEGRATION_TESTS=ON", commands) + for phase in ("integration", "jstests", "tpcc"): + self.assertIn("cmake_ci_runtime.sh build/cmake-ci " + phase, commands) + runtime = (source / ".github/scripts/cmake_ci_runtime.sh").read_text() + self.assertIn("--suites=eloq_basic,eloq_core", runtime) + self.assertIn("--no-tests=error", runtime) + self.assertIn('--server "$build_dir/eloqdoc"', runtime) + shell = (source / ".github/scripts/build_cmake_test_shell.sh").read_text() + self.assertIn("install-shell", shell) + self.assertNotIn("install-core", shell) + self.assertNotIn("install-servers", shell) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/tests/test_server_smoke_config.py b/cmake/tests/test_server_smoke_config.py new file mode 100644 index 0000000000..da94ea3ddd --- /dev/null +++ b/cmake/tests/test_server_smoke_config.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Check smoke-fixture backend selection without a server or cloud service.""" + +import configparser +from pathlib import Path +import unittest + +from server_smoke_config import substrate_config + + +class ServerSmokeConfigTest(unittest.TestCase): + environment = {"S3_ENDPOINT": "http://127.0.0.1:9900/", + "S3_ACCESS_KEY": "test-access", "S3_SECRET_KEY": "test-secret"} + + def config(self, data_store="ELOQDSS_ELOQSTORE", log_state="ROCKSDB_CLOUD_S3", + environment=None, core_number=2): + config = configparser.ConfigParser(interpolation=None) + config.read_string(substrate_config( + Path("/tmp/smoke fixture"), 12345, 12346, data_store, log_state, + self.environment if environment is None else environment, core_number=core_number)) + return config + + def test_eloqstore_s3_uses_same_bucket_and_credentials_for_data_and_log(self): + config = self.config() + local, store = config["local"], config["store"] + bucket = local["txlog_rocksdb_cloud_bucket_name"] + self.assertRegex(bucket, r"^eloqdoc-smoke-[0-9a-f]{32}$") + self.assertTrue(local.getboolean("enable_wal")) + self.assertEqual("", local["txlog_rocksdb_cloud_bucket_prefix"]) + self.assertEqual("txlog", local["txlog_rocksdb_cloud_object_path"]) + self.assertEqual(f"http://127.0.0.1:9900/{bucket}/txlog", + local["txlog_rocksdb_cloud_object_store_service_url"]) + self.assertEqual(local["txlog_rocksdb_cloud_s3_endpoint_url"], + store["eloq_store_cloud_endpoint"]) + self.assertEqual(f"{bucket}/eloqstore", store["eloq_store_cloud_store_path"]) + self.assertEqual("aws", store["eloq_store_cloud_provider"]) + self.assertEqual("test-access", store["aws_access_key_id"]) + self.assertEqual(store["aws_access_key_id"], store["eloq_store_cloud_access_key"]) + self.assertEqual("test-secret", store["aws_secret_key"]) + self.assertEqual(store["aws_secret_key"], store["eloq_store_cloud_secret_key"]) + + def test_cloud_invocations_are_isolated(self): + self.assertNotEqual(self.config()["store"]["eloq_store_cloud_store_path"], + self.config()["store"]["eloq_store_cloud_store_path"]) + + def test_missing_s3_configuration_is_rejected(self): + for key in self.environment: + with self.subTest(key=key): + environment = dict(self.environment, **{key: ""}) + with self.assertRaisesRegex(ValueError, key): + self.config(environment=environment) + + def test_local_rocksdb_needs_no_cloud_service(self): + for data_store in ("ROCKSDB", "ELOQDSS_ROCKSDB"): + with self.subTest(data_store=data_store): + config = self.config(data_store, "ROCKSDB", environment={}) + self.assertFalse(config["local"].getboolean("enable_wal")) + self.assertNotIn("store", config) + self.assertFalse(any("cloud" in key for key in config["local"])) + + def test_unsupported_backend_does_not_silently_use_local_rocksdb(self): + for data_store, log_state in (("ELOQDSS_ELOQSTORE", "ROCKSDB"), + ("ELOQDSS_ROCKSDB", "ROCKSDB_CLOUD_S3"), + ("ELOQDSS_ROCKSDB_CLOUD_S3", "ROCKSDB_CLOUD_S3")): + with self.subTest(data_store=data_store, log_state=log_state): + with self.assertRaisesRegex(ValueError, "Unsupported smoke-test backend"): + self.config(data_store, log_state) + + def test_private_paths_and_ports_are_retained(self): + config = self.config() + self.assertEqual("/tmp/smoke fixture/eloq", config["local"]["eloq_data_path"]) + self.assertEqual("/tmp/smoke fixture/log", config["local"]["log_service_data_path"]) + self.assertEqual(2, config["local"].getint("core_number")) + self.assertEqual(12345, config["local"].getint("tx_port")) + self.assertEqual(12346, config["local"].getint("hm_port")) + self.assertEqual("127.0.0.1:12345", config["cluster"]["tx_ip_port_list"]) + + def test_active_shutdown_can_use_a_single_worker(self): + for data_store, log_state in (("ELOQDSS_ROCKSDB", "ROCKSDB"), + ("ELOQDSS_ELOQSTORE", "ROCKSDB_CLOUD_S3")): + with self.subTest(data_store=data_store): + self.assertEqual(1, self.config(data_store, log_state, core_number=1)[ + "local"].getint("core_number")) + + def test_larger_runtime_suites_can_set_the_memory_budget(self): + config = configparser.ConfigParser() + config.read_string(substrate_config(Path("/tmp/runtime"), 12345, 12346, + "ELOQDSS_ROCKSDB", "ROCKSDB", {}, memory_limit_mb=4000)) + self.assertEqual(4000, config["local"].getint("node_memory_limit_mb")) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/tests/test_server_smoke_diagnostics.py b/cmake/tests/test_server_smoke_diagnostics.py new file mode 100644 index 0000000000..bc4fd3b522 --- /dev/null +++ b/cmake/tests/test_server_smoke_diagnostics.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Check smoke-test failure reporting with real child exits and disposable log files.""" + +import contextlib +import io +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +from server_smoke_diagnostics import describe_exit, report_failure, wait_for_clean_shutdown + + +class SmokeDiagnosticsTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="eloqdoc-smoke-diagnostics-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) / "fixture" + self.root.mkdir() + self.artifacts = Path(self.temp.name) / "artifacts" + self.output = io.StringIO() + + def report(self, returncode, error): + with contextlib.redirect_stderr(self.output): + report_failure(self.root, returncode, error, self.artifacts) + return self.output.getvalue() + + def test_clean_exit_is_required(self): + with subprocess.Popen([sys.executable, "-c", "pass"]) as process: + wait_for_clean_shutdown(process, timeout=5) + self.assertEqual(0, process.returncode) + + def test_nonzero_exit_is_reported_and_full_logs_are_preserved(self): + content = "early log context\n" + "later log context\n" * 250 + "shutdown failure\n" + (self.root / "server.log").write_text(content) + with subprocess.Popen([sys.executable, "-c", "raise SystemExit(23)"]) as process: + with self.assertRaisesRegex(AssertionError, "shutdown failed: exit code 23") as failure: + wait_for_clean_shutdown(process, timeout=5) + output = self.report(process.returncode, failure.exception) + self.assertIn("Server status before fixture cleanup: exit code 23", output) + self.assertIn("shutdown failure", output) + self.assertNotIn("early log context", output) + destination = self.artifacts / self.root.name + self.assertEqual(content, (destination / "server.log").read_text()) + self.assertIn("exit code 23", (destination / "failure.txt").read_text()) + + @unittest.skipUnless(sys.platform != "win32", "POSIX signal exit codes") + def test_signal_exit_has_number_and_name(self): + # SIGTERM exercises signal reporting without generating a core dump. + with subprocess.Popen([sys.executable, "-c", + "import os, signal; os.kill(os.getpid(), signal.SIGTERM)"]) as process: + with self.assertRaisesRegex(AssertionError, r"exit code -15 \(SIGTERM\)"): + wait_for_clean_shutdown(process, timeout=5) + self.assertEqual(-signal.SIGTERM, process.returncode) + self.assertEqual("exit code -11 (SIGSEGV)", describe_exit(-signal.SIGSEGV)) + self.assertEqual("exit code -999 (signal 999)", describe_exit(-999)) + + def test_timeout_is_not_reported_as_a_crash(self): + process = mock.Mock() + process.wait.side_effect = subprocess.TimeoutExpired("server", 30) + with self.assertRaisesRegex(AssertionError, "shutdown timed out after 30s") as failure: + wait_for_clean_shutdown(process, timeout=30) + self.assertIsInstance(failure.exception.__cause__, subprocess.TimeoutExpired) + output = self.report(None, failure.exception) + self.assertIn("Server status before fixture cleanup: still running", output) + self.assertNotIn("SIGTERM", output) + process.terminate.assert_not_called() + process.kill.assert_not_called() + + def test_artifacts_exclude_configs_data_core_files_and_symlinks(self): + included = ("server.log", "replSet.log", "eloqdb.log.FATAL.20260921.123", + "host_manager.log.INFO.20260921.456") + for name in included: + (self.root / name).write_text("test log") + for name in ("data_substrate.cnf", "eloqdoc.yaml", "core.123"): + (self.root / name).write_text("must not upload") + (self.root / "db").mkdir() + (self.root / "db" / "nested.log").write_text("data must not upload") + (self.root / "linked.log").symlink_to(self.root / "data_substrate.cnf") + self.report(-11, AssertionError("crashed")) + self.assertEqual(set(included) | {"failure.txt"}, + {path.name for path in (self.artifacts / self.root.name).iterdir()}) + self.assertNotIn("must not upload", self.output.getvalue()) + + def test_huge_line_and_non_utf8_log_do_not_break_diagnostics(self): + content = b"x" * (128 * 1024) + b"\xfflast line\n" + (self.root / "server.log").write_bytes(content) + output = self.report(1, AssertionError("failure")) + self.assertIn("last line", output) + self.assertLess(len(output), 66 * 1024) + self.assertEqual(content, (self.artifacts / self.root.name / "server.log").read_bytes()) + + def test_unwritable_artifact_destination_still_prints_logs(self): + self.artifacts.write_text("a file blocks directory creation") + (self.root / "server.log").write_text("shutdown stack trace") + output = self.report(1, AssertionError("original failure")) + self.assertIn("Could not save failure summary", output) + self.assertIn("original failure", output) + self.assertIn("shutdown stack trace", output) + + def test_copy_failure_does_not_mask_original_failure(self): + (self.root / "server.log").write_text("shutdown stack trace") + with mock.patch("server_smoke_diagnostics.shutil.copyfile", side_effect=OSError("disk full")): + output = self.report(1, AssertionError("original failure")) + self.assertIn("Could not preserve server.log: disk full", output) + self.assertIn("original failure", output) + self.assertIn("shutdown stack trace", output) + + def test_missing_logs_do_not_mask_original_failure(self): + self.root.rmdir() + output = self.report(1, AssertionError("original failure")) + self.assertIn("Could not list fixture logs", output) + self.assertIn("original failure", output) + + +if __name__ == "__main__": + unittest.main() diff --git a/cmake/tests/test_source_graph.py b/cmake/tests/test_source_graph.py new file mode 100644 index 0000000000..bff60cc134 --- /dev/null +++ b/cmake/tests/test_source_graph.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Regression checks for the standalone server's feature and dependency boundaries.""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from generate_eloqdoc_sources import ( + EXCLUDED_DISTRIBUTED_TEST_TARGETS, load_targets, select_minimal_graph, select_minimal_tests) + + +class StandaloneGraphTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.root = Path(__file__).resolve().parents[2] + cls.targets = load_targets(cls.root / "src") + cls.full_pipeline_sources = set(cls.targets["mongo/db/pipeline/pipeline"]["sources"]) + cls.selected = select_minimal_graph(cls.targets) + cls.sources = { + source for name in cls.selected for source in cls.targets[name]["sources"] + } + + def test_normal_server_features_are_retained(self): + required = { + "mongo/db/dbmain.cpp", "mongo/db/db.cpp", + "mongo/db/service_entry_point_mongod.cpp", + "mongo/db/op_observer_impl.cpp", + "mongo/db/repl/replication_info.cpp", + "mongo/db/auth/authorization_manager_impl.cpp", + "mongo/db/auth/authorization_session_impl.cpp", + "mongo/db/auth/sasl_commands.cpp", "mongo/db/commands/user_management_commands.cpp", + "mongo/db/ttl.cpp", "mongo/db/clientcursor.cpp", + "mongo/db/logical_session_cache_factory_standalone.cpp", + "mongo/db/periodic_runner_job_abort_expired_transactions.cpp", + "mongo/scripting/mozjs/engine.cpp", "mongo/db/commands/mr.cpp", + "mongo/db/commands/shutdown_d.cpp", "mongo/db/commands/fsync.cpp", + "mongo/db/modules/eloq/src/eloq_init.cpp", + } + self.assertFalse(required - self.sources, sorted(required - self.sources)) + + def test_no_embedded_application_or_replica_set_implementation(self): + for source in self.sources: + self.assertFalse(source.startswith("mongo/embedded/"), source) + self.assertNotIn("replication_coordinator_impl", source) + self.assertNotIn("initial_syncer", source) + self.assertNotIn("/bgsync", source) + self.assertNotIn("mongo/scripting/scripting_none.cpp", self.sources) + self.assertNotIn("mongo/s/query/async_results_merger.cpp", self.sources) + self.assertNotIn("mongo/db/pipeline/document_source_merge_cursors.cpp", self.sources) + self.assertNotIn("mongo/client/parallel.cpp", self.sources) + self.assertNotIn("mongo/s/client/shard_connection.cpp", self.sources) + self.assertNotIn("mongo/db/logical_session_cache_factory_mongod.cpp", self.sources) + self.assertNotIn("mongo/db/sessions_collection_sharded.cpp", self.sources) + self.assertNotIn("mongo/db/sessions_collection_rs.cpp", self.sources) + + def test_collations_use_scons_pinned_icu_and_data(self): + self.assertTrue({ + "third_party/icu4c-57.1/source/icu_common", + "third_party/icu4c-57.1/source/icu_i18n", + "third_party/icu4c-57.1/source/icu_data", + } <= self.selected) + self.assertIn("mongo/util/icu_init.cpp", self.sources) + self.assertNotIn("mongo/util/icu_init_stub.cpp", self.sources) + + def test_integration_tests_retain_startup_initializers(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + integration = [test for test in tests if test["kind"] == "integration"] + self.assertEqual(7, len(integration)) + for test in integration: + with self.subTest(name=test["name"]): + self.assertIn("mongo/util/options_parser/options_parser_init.cpp", + test["support_sources"]) + self.assertIn("mongo/util/version_impl.cpp", test["support_sources"]) + + def test_icu_tests_retain_embedded_data_initializer(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + by_name = {test["name"]: test for test in tests} + for name in ("mongo/util/icu_test", "mongo/db/query/collation/collator_factory_icu_test", + "mongo/db/query/collation/collator_interface_icu_test"): + with self.subTest(name=name): + self.assertIn("mongo/util/icu_init.cpp", by_name[name]["support_sources"]) + + def test_sharded_pipeline_is_only_in_full_build(self): + common = "mongo/db/pipeline/pipeline.cpp" + sharded = {"mongo/db/pipeline/pipeline_sharded.cpp", + "mongo/db/pipeline/cluster_aggregation_planner.cpp"} + self.assertIn(common, self.full_pipeline_sources) + self.assertTrue(sharded <= self.full_pipeline_sources) + self.assertIn(common, self.sources) + self.assertFalse(sharded & self.sources) + + tests, support = select_minimal_tests(self.targets, self.selected) + standalone_sources = self.sources | support | { + source for test in tests for source in test["sources"]} + self.assertFalse(sharded & standalone_sources) + self.assertNotIn("mongo/db/pipeline/cluster_aggregation_planner_standalone.cpp", + standalone_sources) + + def test_tests_cover_server_but_not_data_substrate(self): + tests, support = select_minimal_tests(self.targets, self.selected) + required = { + "mongo/db/standalone_runtime_test", + "mongo/db/query/get_executor_test", + "mongo/db/commands/plan_cache_commands_test", + "mongo/db/commands/index_filter_commands_test", + "mongo/db/storage/ephemeral_for_test/storage_ephemeral_for_test_record_store_test", + } + self.assertFalse(required - {test["name"] for test in tests}) + for source in support | {src for test in tests for src in test["sources"]}: + self.assertNotIn("/data_substrate/", source) + + def test_replication_suites_are_excluded(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + for test in tests: + self.assertFalse(test["name"].startswith("mongo/db/repl/"), test["name"]) + + def test_distributed_suites_are_excluded_without_dropping_shared_coverage(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + names = {test["name"] for test in tests} + excluded = set(EXCLUDED_DISTRIBUTED_TEST_TARGETS) + self.assertTrue(excluded <= self.targets.keys()) + self.assertFalse(excluded & names) + for name in names: + self.assertFalse(name.startswith(("mongo/db/repl/", "mongo/db/s/", "mongo/s/")), name) + # Keep local request lifecycle, wire-format, metadata, and catalog coverage. + self.assertTrue({ + "mongo/db/operation_context_test", + "mongo/db/standalone_runtime_test", + "mongo/db/logical_time_test", + "mongo/db/logical_session_cache_test", + "mongo/db/catalog/collection_options_test", + "mongo/db/catalog/uuid_catalog_test", + "mongo/rpc/client_metadata_test", + "mongo/rpc/rpc_metadata_test", + "mongo/client/read_preference_test", + } <= names) + + def test_auth_tests_retain_name_registered_implementations(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + auth = next(test for test in tests + if test["name"] == "mongo/db/auth/authorization_session_test") + self.assertIn("mongo/db/auth/authorization_manager_global.cpp", auth["support_sources"]) + self.assertIn("mongo/db/pipeline/document_source_lookup.cpp", auth["support_sources"]) + self.assertIn("mongo/db/pipeline/document_source_facet.cpp", auth["support_sources"]) + + def test_query_tests_retain_authorization_factories(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + required = { + "mongo/db/auth/authorization_manager_global.cpp", + "mongo/db/auth/authorization_manager_impl.cpp", + "mongo/db/auth/authorization_session_impl.cpp", + } + for name in ("mongo/db/query/get_executor_test", + "mongo/db/commands/plan_cache_commands_test", + "mongo/db/commands/index_filter_commands_test", + "mongo/db/operation_context_test"): + with self.subTest(name=name): + test = next(test for test in tests if test["name"] == name) + self.assertFalse(required - set(test["support_sources"])) + + def test_test_only_engine_and_parser_registrations(self): + tests, _ = select_minimal_tests(self.targets, self.selected) + by_name = {test["name"]: test for test in tests} + engine = "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_init.cpp" + self.assertNotIn(engine, self.sources) + self.assertIn(engine, by_name["mongo/db/exec/sort_test"]["support_sources"]) + for name in ("mongo/db/exec/sort_test", "mongo/db/exec/queued_data_stage_test", + "mongo/db/concurrency/lock_manager_test"): + self.assertIn("mongo/db/catalog/database_holder_impl.cpp", + by_name[name]["support_sources"]) + self.assertIn("mongo/db/catalog/collection_impl.cpp", by_name[name]["support_sources"]) + self.assertIn("mongo/db/pipeline/document_source_internal_split_pipeline.cpp", + by_name["mongo/db/pipeline/document_source_facet_test"]["support_sources"]) + self.assertIn("mongo/db/pipeline/granularity_rounder_powers_of_two.cpp", + by_name["mongo/db/pipeline/granularity_rounder_test"]["support_sources"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/how-to-compile.md b/docs/how-to-compile.md index 139b1d473b..27d53e9aa8 100644 --- a/docs/how-to-compile.md +++ b/docs/how-to-compile.md @@ -1,73 +1,354 @@ -# How To Compile EloqDoc-RocksDB +# Build EloqDoc -```bash -git clone --recurse-submodules https://github.com/eloqdata/eloqdoc -``` +EloqDoc supports two build systems in the same source tree: + +- The top-level CMake/Ninja build is the recommended server-only build. It builds the + MongoDB-compatible single-node server, the Eloq storage-engine integration, and the in-tree Data + Substrate in one graph. It uses Python 3 and does not invoke SCons. +- The historical MongoDB 4.0 SCons build remains supported for existing development and packaging + workflows. It uses Python 2.7 and builds the Eloq core libraries separately with CMake. + +Use separate build directories and installation prefixes if both builds are used from one checkout. -## 0. Install dependencies +The top-level CMake server uses the existing EloqDoc `dbmain.cpp` / `db.cpp` startup and +`ServiceEntryPointMongod` request path. It retains authentication and authorization, TTL and +background indexes, background maintenance, session cleanup, diagnostics, and the MozJS +JavaScript engine. It does not use the embedded application runtime. -Assuming Ubuntu 24.04 is your development environment, run the following script to install dependencies. +Both builds use the vendored ICU 57.1 implementation and the same embedded collation data. +CMake does not select the host's ICU: its version is recorded in index collation metadata, +so switching to a newer system version would change compatibility with SCons-created indexes. +Existing Eloq storage-engine restrictions still apply; for example, `mapReduce` remains explicitly +unsupported even though server-side JavaScript such as `$where` is available. +The test-only `restartCatalog` command is also rejected for Eloq in both builds: its no-op MongoDB +locker cannot guarantee that other workers stop using catalog entries during a live reload. + +The build excludes `mongos`, MongoDB replica-set coordination and initial sync, WiredTiger, +the MMAPv1 and Mobile engine registrations, and distributed MongoDB aggregation. A standalone +implementation supplies the shared replication interface without starting replica-set workers; +Eloq Data Substrate continues to provide its own distributed transactions and durability. +Sharding-only pipeline methods live in `pipeline_sharded.cpp`, which SCons builds alongside the +original distributed aggregation planner. CMake omits both files; the shared `pipeline.cpp` +handles local aggregation without distributed-planner rejection stubs. +Some `repl/`, `s/`, and MMAPv1 sources remain because MongoDB's +shared command/catalog code uses their wire types, interfaces, and legacy B-tree helpers; the +distributed runtimes and alternate storage backends themselves are excluded and audited during +configuration. + +## 1. Get the source and dependencies ```bash -mkdir -p /tmp/eloqdoc-deps +git clone --recurse-submodules https://github.com/eloqdata/eloqdoc +cd eloqdoc bash scripts/install_dependency_ubuntu2404.sh /tmp/eloqdoc-deps ``` -If you are using another Linux distribution, follow the steps in `install_dependency_ubuntu2404.sh` to install dependencies manually. +The dependency script supports Ubuntu 24.04. It installs CMake, Ninja, Python 3, ICU, Python 2.7 +with the historical SCons requirements, and the Data Substrate dependency prefix. Set +`ELOQ_INSTALL_LEGACY_SCONS=0` when preparing a CMake-only environment. On another Linux +distribution, install equivalent packages and build the dependencies described by +`src/mongo/db/modules/eloq/data_substrate/scripts/third_party/install-ubuntu2404.sh`. -EloqDoc still uses the MongoDB 4.0 `scons` build, which requires Python 2.7 and its build-time packages. Ubuntu 24.04 does not provide Python 2.7 through the default apt repositories, so the dependency script installs Python 2.7.18 with `pyenv`. If you use a custom image or install dependencies manually, make sure `python2` and `pip2` are available on `PATH` before building. +The CMake IDL compiler requires PyYAML in the Python 3 interpreter selected by CMake. +For Ubuntu's `/usr/bin/python3`, install it with `sudo apt-get install python3-yaml` +(the dependency script already does this). For a virtual environment, install `PyYAML` +there and pass `-DPython3_EXECUTABLE=/path/to/venv/bin/python` to CMake. Python 2's +SCons packages and the cached C++ dependency prefix do not provide this Python 3 module. -```bash -python2 --version -pip2 --version +By default the Data Substrate dependencies are installed at: + +```text +src/mongo/db/modules/eloq/data_substrate/third_party/install ``` -## 1. Compile EloqDoc-RocksDB +Set `ELOQ_THIRD_PARTY_PREFIX` before running the installer and either build system to use a different +absolute prefix. + +Use a private dependency prefix, not `/usr`: the SCons flags and CI helper add +`-isystem /include`. Adding `/usr/include` this way can break GCC's +`#include_next` lookup in the C++ standard-library headers. -There are two components to compile: EloqDoc and core libraries. EloqDoc is compiled with `scons`, and the core libraries are compiled with `cmake`. +## 2. Build with top-level CMake and Ninja -### 1.1 Define an installation path +From the repository root: ```bash export INSTALL_PREFIX=/absolute/path/to/install +export ELOQ_THIRD_PARTY_PREFIX="$PWD/src/mongo/db/modules/eloq/data_substrate/third_party/install" + +cmake -S . -B build/cmake -G Ninja \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_PREFIX" \ + -DELOQ_THIRD_PARTY_PREFIX="$ELOQ_THIRD_PARTY_PREFIX" \ + -DWITH_DATA_STORE=ELOQDSS_ROCKSDB \ + -DWITH_LOG_STATE=ROCKSDB + +cmake --build build/cmake -j8 +cmake --install build/cmake ``` -The `INSTALL_PREFIX` must be an absolute path. Binaries and libraries will be installed under this directory. +To select GCC 15, add `-DCMAKE_C_COMPILER=gcc-15 -DCMAKE_CXX_COMPILER=g++-15` to +the configure command and use a separate build directory such as `build/cmake-gcc15`. +Do not change compilers inside an existing build directory; keep the GCC 13, GCC 15, +and SCons build trees separate. Selecting GCC 15 keeps the MongoDB sources in C++17 mode. -### 1.2 Compile core libraries +The server is installed as `$INSTALL_PREFIX/bin/eloqdoc`. Eloq is the default and only registered +storage engine; explicitly selecting another engine is rejected. MongoDB replica-set, shard-server, +and config-server startup options are rejected. Normal server options, including `--auth`, +`--config`, and `--fork`, use the existing server parser and startup handling. -Run the following commands from the repository root: +The current Linux profile builds MongoDB without client-facing TLS, matching the historical SCons +configuration documented below. Data Substrate's separate TLS dependencies are unaffected. -```bash -cmake -S src/mongo/db/modules/eloq \ - -B src/mongo/db/modules/eloq/build \ - -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \ - -DWITH_DATA_STORE=ELOQDSS_ROCKSDB -cmake --build src/mongo/db/modules/eloq/build -j8 -cmake --install src/mongo/db/modules/eloq/build -``` +IDL generation uses per-output depfiles, including transitive imports, so editing one IDL file +regenerates only its dependent outputs. The historical SCons IDL compiler is unchanged. -### 1.3 Compile EloqDoc-RocksDB +Common data-store choices are: -The `scons` build tool depends on Python 2.7. Switch to a Python 2.7 environment before running `scons`. The `install_dependency_ubuntu2404.sh` script installs Python 2.7.18 and the required packages. +| `WITH_DATA_STORE` | Matching `WITH_LOG_STATE` | +|---|---| +| `ROCKSDB` or `ELOQDSS_ROCKSDB` | `ROCKSDB` | +| `ELOQDSS_ROCKSDB_CLOUD_S3` | `ROCKSDB_CLOUD_S3` | +| `ELOQDSS_ROCKSDB_CLOUD_GCS` | `ROCKSDB_CLOUD_GCS` | +| `ELOQDSS_ELOQSTORE` | `ROCKSDB`, `ROCKSDB_CLOUD_S3`, or `ROCKSDB_CLOUD_GCS` | -```bash -pyenv global 2.7.18 -``` +To make an AddressSanitizer build, add `-DELOQDOC_ENABLE_ASAN=ON`. + +## 3. Build with SCons -Compile EloqDoc from the repository root. +The SCons path first builds and installs the Eloq core libraries, then runs MongoDB's historical +build. Use an installation prefix separate from the top-level CMake build: ```bash +export SCONS_INSTALL_PREFIX=/absolute/path/to/scons-install +export ELOQ_THIRD_PARTY_PREFIX="$PWD/src/mongo/db/modules/eloq/data_substrate/third_party/install" + +cmake -S src/mongo/db/modules/eloq \ + -B build/eloq-scons-deps \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX="$SCONS_INSTALL_PREFIX" \ + -DELOQ_THIRD_PARTY_PREFIX="$ELOQ_THIRD_PARTY_PREFIX" \ + -DWITH_DATA_STORE=ELOQDSS_ROCKSDB \ + -DWITH_LOG_STATE=ROCKSDB +cmake --build build/eloq-scons-deps -j8 +cmake --install build/eloq-scons-deps + +pyenv global 2.7.18 env WITH_DATA_STORE=ELOQDSS_ROCKSDB \ -python scripts/buildscripts/scons.py \ + WITH_LOG_STATE=ROCKSDB \ + ELOQ_THIRD_PARTY_PREFIX="$ELOQ_THIRD_PARTY_PREFIX" \ +python2 scripts/buildscripts/scons.py \ MONGO_VERSION=4.0.3 \ VARIANT_DIR=RelWithDebInfo \ - CXXFLAGS="-Wno-nonnull -Wno-class-memaccess -Wno-interference-size -Wno-redundant-move" \ - --build-dir=#build \ - --prefix=$INSTALL_PREFIX \ + CXXFLAGS="-isystem $ELOQ_THIRD_PARTY_PREFIX/include -Wno-nonnull -Wno-class-memaccess -Wno-interference-size -Wno-redundant-move" \ + CPPDEFINES="ELOQ_MODULE_ENABLED EXT_TX_PROC_ENABLED" \ + LIBPATH="$ELOQ_THIRD_PARTY_PREFIX/lib $ELOQ_THIRD_PARTY_PREFIX/lib64" \ + --build-dir=#build/scons \ + --prefix="$SCONS_INSTALL_PREFIX" \ + --allocator=system \ + --link-model=dynamic \ + --install-mode=hygienic \ --disable-warnings-as-errors \ -j8 \ install-core ``` -All executable files will be installed to `$INSTALL_PREFIX/bin`, and all libraries will be installed to `$INSTALL_PREFIX/lib`. +This path retains the existing SCons target definitions and does not define the CMake-only +`ELOQDOC_STANDALONE` compile definition. + +### SCons native-test validation profile + +The historical `all`/`install-tests` aliases include unsupported engines, embedded executables, +and upstream tests with obsolete fork interfaces. For EloqDoc validation, filter the generated +unit-test manifest explicitly (Python 3 is used only for this selector; SCons/resmoke use Python 2): + +```bash +python3 scripts/buildscripts/select_eloq_unit_tests.py \ + --manifest build/scons/unittests.txt \ + --output build/scons/eloq-unittests.txt \ + --report build/scons/eloq-unittest-selection.json \ + --suite build/scons/eloq-unittests.yml +``` + +Build the binary paths listed in `eloq-unittests.txt` as explicit SCons targets, using the same +compiler/options/build directory as the server command above, instead of `all` or `install-tests`. +For example, in Bash, `mapfile -t test_targets < build/scons/eloq-unittests.txt` supplies the +targets as `"${test_targets[@]}"`. Then run: + +```bash +python2 scripts/buildscripts/resmoke.py \ + --suites=build/scons/eloq-unittests.yml --jobs=4 --continueOnFailure +``` + +The selection report records each skipped target and its reason. MongoDB replication tests +(`mongo/db/repl/`), MMAPv1, WiredTiger, embedded tests, and individually listed obsolete +interfaces/fixtures are excluded; legacy `dbtest` is not part of this profile. Missing retained +binaries are **not** automatically skipped. Keep the build/install +library directories on `LD_LIBRARY_PATH` for dynamic SCons tests, including test-helper libraries +that are not installed by `install-core`. Limit build/test parallelism to eight; on memory-limited +WSL hosts, use two compiler jobs and one linker job (`-j2 --jlink=1`). CPU affinity (`taskset -c 0-7` +on a host with these CPUs available) also bounds internal worker threads. + +The `ephemeralForTest` fixture implements the fork's namespace-keyed catalog metadata, +enumeration, collection lookup/version, and cursor ownership interfaces. Its index cursors also +return the documents required by Eloq's fast `_id` lookup. This is a test-only in-memory model, +not a test of distributed storage. Native regression tests cover catalog commit/rollback, +duplicate namespaces, feature documents, and index lookup with a prefetched document. + +All suites under `mongo/db/repl/` are skipped, including replication-process, consistency-marker, +drop-pending reaper, replica-set coordinator, and initial-sync tests. This matches CMake's existing +replication-test exclusion; it changes validation selection, not the historical SCons targets or +the shared interfaces used by the standalone server. + +Outside that directory, confirmed fixtures requiring MongoDB's replication oplog, local-shard +locking, capped conversion, or removed local-catalog rename/ident-repair APIs are excluded by exact +target in `scripts/buildscripts/select_eloq_unit_tests.py`. These are unsupported/obsolete fixtures, +not WiredTiger failures and not repaired tests. There is no blanket catalog or sharding-directory +exclusion: collection metadata, KV collection catalog, repair observer, and free-monitoring tests +remain selected. New or unexplained failures outside the documented exclusions remain failures. +Do not remove production `MONGO_UNREACHABLE` guards or return empty mock catalogs to make obsolete +fixtures pass. + +The nesting-depth integration test runs with either engine: it expects Eloq update failures at +command level (command-level transaction rollback), and upstream-engine failures in `writeErrors`. +Both branches check `Overflow` and verify that the rejected update leaves the document unchanged. + +## 4. Build and run CMake tests + +Enable the tests when configuring the same build directory: + +```bash +cmake -S . -B build/cmake -G Ninja \ + -DELOQDOC_BUILD_TESTS=ON + +cmake --build build/cmake --target eloqdoc-tests -j8 +ctest --test-dir build/cmake --output-on-failure -j8 +``` + +The generated test graph contains the C++ unit and integration-test declarations belonging to the +retained single-node MongoDB modules, plus the Eloq storage-integration tests declared by the Eloq +Mongo module. Tests for `mongos`, replication runtimes, and excluded storage engines are omitted. +The CMake-only exclusions also cover distributed-only suites outside those directories: +replica-set clients/monitors, cluster-time signing keys/proofs, remote operation-time tracking, +and config-server/replication RPC metadata. Shared wire-format and local catalog/session tests +remain enabled. `operation_context_test` covers tracker reuse during context recycling (reuse +when exclusively owned; preserve the old tracker and replace it when shared), without enabling +the distributed-execution suites. These exclusions do not change SCons target definitions or +its validation selector. +The in-memory `ephemeralForTest` engine is linked only into unit-test fixtures that need a real +MongoDB storage engine and locker; it is not registered in the CMake server. +Data Substrate and data-store unit tests are also deliberately disabled: those components keep and +run their tests in their own repositories. + +MongoDB integration tests require an already-running EloqDoc fixture, so their binaries are built +but their server-dependent cases are not registered with CTest by default. Every integration +binary has an always-registered `startup-options` check which runs `--help` without a server; +this catches missing option-parser initializers without a server. To register the full cases, +configure with: + +```bash +-DELOQDOC_REGISTER_INTEGRATION_TESTS=ON \ +-DELOQDOC_TEST_CONNECTION_STRING=localhost:27017 +``` + +Then run `ctest --test-dir build/cmake --label-regex integration --output-on-failure` after starting +the fixture. + +GitHub's CMake job explicitly enables `ELOQDOC_REGISTER_INTEGRATION_TESTS`. It runs unit and +integration-startup checks first, then runs the actual C++ integration suites against a managed +CMake-built server. These suites execute serially because they modify shared server parameters +and failpoints. The same job runs the existing `eloq_basic` and `eloq_core` JavaScript suites, +the same two-warehouse transactional TPCC workload as SCons CI, and the wire-protocol smoke tests. +The CI matrix remains one amd64 RelWithDebInfo EloqStore/S3 configuration for CMake. + +The legacy JavaScript client is built separately with SCons's `install-shell` target; this does +not build or substitute a SCons server. Python 2 is needed for that shell build and resmoke, +while the server fixture uses Python 3/PyMongo. Their Python package paths are isolated. +Each runtime phase gets fresh local data and a private S3 bucket on its disposable RustFS +service, and must pass both its tests and clean server shutdown. Failure logs are preserved in +`build/cmake-ci/runtime-diagnostics/`; configs, credentials, and data directories are not uploaded. + +To reproduce individual CI runtime phases locally (cloud configurations also need RustFS on +`PATH` and the S3 environment variables documented below): + +```bash +cmake -S . -B build/cmake -DELOQDOC_REGISTER_INTEGRATION_TESTS=ON \ + -DELOQDOC_TEST_CONNECTION_STRING=localhost:27017 +cmake --build build/cmake --target eloqdoc eloqdoc-tests -j8 +bash .github/scripts/cmake_ci_runtime.sh build/cmake integration +ELOQDOC_TEST_SHELL=/path/to/eloqdoc-cli \ + bash .github/scripts/cmake_ci_runtime.sh build/cmake jstests +PY_TPCC_PATH=/path/to/py-tpcc \ + bash .github/scripts/cmake_ci_runtime.sh build/cmake tpcc +``` + +The fixture interpreter defaults to `/usr/bin/python3` and can be selected with +`CMAKE_TEST_PYTHON`; install `pymongo==4.8.0` for that interpreter. TPCC uses its own virtual +environment. The JavaScript and TPCC phases reserve port 27017 and refuse to reuse an existing +server on that port. No Data Substrate/data-store test suites are added. + +The existing `eloq_basic` and `eloq_core` JavaScript suites can also run against the CMake +server using an EloqDoc shell. Put the shell directory on `PATH` as well as passing resmoke's +`--mongo` option, because some tests spawn `eloqdoc-cli` by name. `set_param1.js` always runs +its generic parameter checks and runs its oplog-fetcher checks only when the server advertises +replication commands. This preserves those assertions for SCons without restoring replication +to the CMake server or excluding the whole test. + +The server smoke test uses the build's selected storage/log backend. It supports local +`ELOQDSS_ROCKSDB` (or `ROCKSDB`) with `ROCKSDB` logging and WAL disabled, or +`ELOQDSS_ELOQSTORE` with `ROCKSDB_CLOUD_S3` logging and WAL enabled. Both start an +authenticated server with temporary local storage and at most two Substrate worker cores. +EloqDoc's catalog requires a data-store handler even for a disposable fixture. The test checks +CRUD, ICU 57.1 collation metadata and lookup, local aggregation, JavaScript, +background/TTL indexes, sessions and transaction commit/abort, +adaptive ingress reactors and coroutine request scheduling, unsupported-topology rejection, +maintenance-mode command dispatch, getMore term validation, fsync, diagnostics, and clean shutdown +while a long-running command is active. It enables test-only commands to +check view-catalog invalidation and safe rejection of online catalog restart; compact, touch, and +repairDatabase must still report Eloq's existing storage-engine limitations. Install the test-only +client and run: + +```bash +python3 -m pip install --target build/cmake/python 'pymongo==4.8.0' +PYTHONPATH="$PWD/build/cmake/python" \ + cmake --build build/cmake --target eloqdoc-server-smoke -j8 +``` + +For EloqStore/S3, first start a disposable S3-compatible service and export `S3_ENDPOINT`, +`S3_ACCESS_KEY`, and `S3_SECRET_KEY`. The fixture uses a new bucket on each run and retains +it for inspection; stop/remove your disposable service afterward. CI starts and stops its own +RustFS instance. The CMake CI job tests only amd64/RelWithDebInfo/EloqStore-S3; the SCons CI +jobs cover amd64/RelWithDebInfo with RocksDB-S3 and EloqStore-S3. + +The fixture is restricted to at most eight CPUs and retains its logs and data in the temporary +directory printed at startup. It tests Mongo-facing behavior, not Data Substrate/data-store +persistence or recovery. Use `--option=value` for server options with values, as in the existing +SCons launch scripts: the gflags pass can reorder separate argument values. + +On a runtime failure, the smoke test reports the server's exit code (and signal name when +applicable) before cleanup, prints bounded log tails, and copies full fixture logs plus a failure +summary to `build/cmake/smoke-diagnostics/`. CI uploads these diagnostics and the RustFS log +as a seven-day artifact. Data directories and configuration files are not uploaded. A shutdown +timeout remains distinct from a nonzero exit, and clean shutdown still requires exit code zero. + +The smoke target runs twice to verify TTL shutdown ordering: once with the worker in a +600-second sleep (which shutdown must wake promptly), and once with an active pass paused +before storage access. Both require the TTL worker to finish before storage teardown, while +also interrupting a long-running client command. The active-pass run uses a single Substrate +worker so a blocking wait on the shutdown coroutine would strand TTL's requests. It uses the test-only +`hangTTLMonitorBeforeStorageAccess` failpoint; it is inactive during normal operation. + +## 5. Inspect the selected MongoDB graph + +The source list is derived statically from MongoDB's fine-grained `SConscript` library metadata; +SCons itself is never imported or executed. This keeps the CMake graph synchronized with the +existing module declarations while allowing explicit single-node substitutions. + +```bash +cmake --build build/cmake --target eloqdoc-source-manifest +``` + +For source/compiler validation on a machine without the Data Substrate dependency prefix, configure +with `-DELOQDOC_BUILD_DATA_SUBSTRATE=OFF`. Generated-source and individual Mongo object targets can +then be built, but the final executable is deliberately not linkable in that mode. diff --git a/scripts/buildscripts/select_eloq_unit_tests.py b/scripts/buildscripts/select_eloq_unit_tests.py new file mode 100644 index 0000000000..3b18010c4f --- /dev/null +++ b/scripts/buildscripts/select_eloq_unit_tests.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Select SCons native tests for EloqDoc, with an explicit, auditable skip report. + +This filters the SCons unit-test manifest, not the SCons target definitions. Unknown or +missing binaries are retained: a build failure must not silently turn into a test skip. +Only confirmed unsupported engines, runtimes, and legacy fixtures are excluded. +""" + +import argparse +import json +from pathlib import Path + + +EXCLUDED_PREFIXES = { + "mongo/db/repl/": "MongoDB replication tests are outside EloqDoc validation", + "mongo/db/storage/mmap_v1/": "MMAPv1 is not an EloqDoc storage engine", + "mongo/db/storage/wiredtiger/": "WiredTiger is not an EloqDoc storage engine", + "mongo/embedded/": "embedded runtime is not the EloqDoc server", +} + +# These upstream test targets still use interfaces removed by the Eloq fork. Do not +# exclude their whole directories: the other query, pipeline, and catalog tests matter. +OBSOLETE_TARGETS = { + "mongo/db/catalog/rename_collection_test": "old Collection ownership interface", + "mongo/db/commands/mr_test": "old Collection ownership interface", + "mongo/db/pipeline/document_source_test": "change-stream test uses old Collection ownership", + "mongo/db/sessions_test": "old Session constructor", + "mongo/db/storage/ephemeral_for_test/storage_ephemeral_for_test_engine_test": + "KV harness uses removed catalog namespace interface", + "mongo/db/storage/kv/kv_database_catalog_entry_test": + "removed catalog namespace interface", + "mongo/db/storage/mobile/storage_mobile_kv_engine_test": + "KV harness uses removed catalog namespace interface", + "mongo/db/views/views_test": "removed catalog namespace interface", + "mongo/s/query/cluster_find_test": "old pooled query ownership interface", + "mongo/s/sharding_routing_table_test": "old query/namespace test interface", +} + +# These fixtures create MongoDB's replication oplog during setup. EloqDoc does not +# implement that oplog; changing production catalog semantics to run them would be +# incorrect. Outside the excluded repl/ directory, keep exact targets rather than +# excluding all of s/ or catalog/: those directories retain applicable coverage. +UNSUPPORTED_RUNTIME_FIXTURES = dict.fromkeys(( + "mongo/db/catalog/database_test", + "mongo/db/catalog/drop_database_test", + "mongo/db/keys_collection_manager_sharding_test", + "mongo/db/logical_clock_test", + "mongo/db/logical_time_validator_test", + "mongo/db/op_observer_impl_test", + "mongo/db/ops/write_ops_retryability_test", + "mongo/db/s/balancer_test", + "mongo/db/s/collection_sharding_runtime_test", + "mongo/db/s/config_server_op_observer_test", + "mongo/db/s/session_catalog_migration_destination_test", + "mongo/db/s/session_catalog_migration_source_test", + "mongo/db/s/shard_server_test", + "mongo/db/s/sharding_catalog_manager_test", + "mongo/db/transaction_history_iterator_test", + "mongo/s/catalog/replset_dist_lock_manager_test", +), "fixture requires MongoDB's replication oplog") +UNSUPPORTED_RUNTIME_FIXTURES.update({ + "mongo/db/catalog/capped_utils_test": + "capped collection conversion is explicitly unsupported by EloqDoc", + "mongo/db/catalog/create_collection_test": + "fixture uses removed local-catalog renameCollection API", + "mongo/db/storage/kv/kv_storage_engine_test": + "fixture uses removed local-catalog ident reconciliation/repair APIs", + "mongo/s/client/shard_local_test": + "MongoDB local-shard fixture assumes MongoDB write-unit/lock semantics", +}) + + +def target_name(binary): + parts = Path(binary).parts + # The manifest can contain absolute paths or build-directory-relative paths. + try: + return "/".join(parts[parts.index("mongo"):]) + except ValueError: + return binary + + +def skip_reason(binary): + name = target_name(binary) + for prefix, reason in EXCLUDED_PREFIXES.items(): + if name.startswith(prefix): + return reason + return OBSOLETE_TARGETS.get(name) or UNSUPPORTED_RUNTIME_FIXTURES.get(name) + + +def select_tests(binaries): + selected, skipped = [], [] + for binary in binaries: + reason = skip_reason(binary) + if reason: + skipped.append({"test": binary, "reason": reason}) + else: + selected.append(binary) + return selected, skipped + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--report", required=True, type=Path) + parser.add_argument("--suite", type=Path, help="also write a resmoke suite (JSON/YAML)") + args = parser.parse_args() + binaries = [line.strip() for line in args.manifest.read_text().splitlines() if line.strip()] + selected, skipped = select_tests(binaries) + args.output.write_text("".join(binary + "\n" for binary in selected)) + args.report.write_text(json.dumps({"selected": selected, "skipped": skipped}, indent=2) + "\n") + if args.suite: + args.suite.write_text(json.dumps({ + "test_kind": "cpp_unit_test", + "selector": {"root": str(args.output.resolve())}, + "executor": {"config": {}}, + }, indent=2) + "\n") + print("Selected %d tests; skipped %d (reasons: %s)" % + (len(selected), len(skipped), args.report)) + + +if __name__ == "__main__": + main() diff --git a/scripts/install_dependency_ubuntu2404.sh b/scripts/install_dependency_ubuntu2404.sh index 4c68c1975a..77ea7f2c3c 100755 --- a/scripts/install_dependency_ubuntu2404.sh +++ b/scripts/install_dependency_ubuntu2404.sh @@ -7,11 +7,13 @@ set -euo pipefail # scripts/install_dependency_ubuntu2404.sh [TEMP_DIR] [--skip_eloq_common] # # TEMP_DIR controls temporary downloads and build artifacts. It defaults to -# /tmp/eloqdoc-deps. The legacy --skip_eloq_common option installs only the -# Python 2.7 build environment. +# /tmp/eloqdoc-deps. The legacy --skip_eloq_common option skips the CMake and +# Data Substrate dependencies. # # Set ELOQ_SKIP_THIRD_PARTY=1 to skip the Data Substrate third-party workspace # build. CI uses that mode when the third-party prefix is restored from cache. +# Python 2 and the SCons requirements are installed by default so the historical build keeps +# working. Set ELOQ_INSTALL_LEGACY_SCONS=0 for a smaller CMake-only environment. SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) REPO_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) @@ -132,7 +134,7 @@ install_python2() { fi if ! command -v python2 >/dev/null 2>&1; then - log_info "Installing Python 2.7.18 with pyenv" + log_info "Installing Python 2.7.18 with pyenv for the legacy SCons workflow" apt_install \ build-essential zlib1g-dev libbz2-dev liblzma-dev libreadline-dev \ libsqlite3-dev libffi-dev libssl-dev libncurses5-dev \ @@ -202,13 +204,13 @@ configure_timezone if [ "${SKIP_ELOQ_COMMON}" = false ]; then log_info "Installing EloqDoc system build packages" system_packages=( - sudo wget curl apt-utils python3 python3-dev python3-pip python3-venv + sudo wget curl apt-utils python3 python3-dev python3-pip python3-venv python3-yaml libcurl4-openssl-dev build-essential libncurses5-dev libncursesw5-dev gnutls-dev bison zlib1g-dev ccache rsync cmake ninja-build libuv1-dev git g++ gcc make openssh-client libssl-dev libgflags-dev libleveldb-dev libsnappy-dev openssl libbz2-dev liblz4-dev libzstd-dev libboost-context-dev ca-certificates libc-ares-dev libc-ares2 m4 - pkg-config tar xz-utils libreadline-dev libsqlite3-dev ncurses-dev + pkg-config tar xz-utils libreadline-dev libsqlite3-dev ncurses-dev libicu-dev tk-dev libffi-dev liblzma-dev patchelf libprotobuf-dev protobuf-compiler libjsoncpp-dev unzip ) @@ -230,6 +232,10 @@ else log_info "Skipping EloqDoc system and Data Substrate third-party dependencies" fi -install_python2 +if [ "${ELOQ_INSTALL_LEGACY_SCONS:-1}" = "1" ]; then + install_python2 +else + log_info "Skipping the Python 2/SCons environment (ELOQ_INSTALL_LEGACY_SCONS=0)" +fi log_info "EloqDoc dependency installation completed successfully" diff --git a/src/mongo/base/SConscript b/src/mongo/base/SConscript index 27c0bbf756..b0a47a68e6 100644 --- a/src/mongo/base/SConscript +++ b/src/mongo/base/SConscript @@ -40,6 +40,8 @@ env.CppUnitTest('base_test', 'data_type_validated_test.cpp', 'data_view_test.cpp', 'encoded_value_storage_test.cpp', + 'error_codes_test.cpp', + 'error_codes_test_helper.cpp', 'initializer_dependency_graph_test.cpp', 'initializer_test.cpp', 'owned_pointer_map_test.cpp', diff --git a/src/mongo/base/error_codes.tpl.h b/src/mongo/base/error_codes.tpl.h index 472c6d72f7..f536906e8a 100644 --- a/src/mongo/base/error_codes.tpl.h +++ b/src/mongo/base/error_codes.tpl.h @@ -121,11 +121,13 @@ inline bool ErrorCodes::isA(Error code) { */ namespace error_details { +// These definitions live in a header, including the explicit specializations below. +// Inline variables keep a single definition across translation units. template -constexpr bool isNamedCode = false; +inline constexpr bool isNamedCode = false; //#for $ec in $codes template <> -constexpr bool isNamedCode = true; +inline constexpr bool isNamedCode = true; //#end for MONGO_COMPILER_NORETURN void throwExceptionForStatus(const Status& status); diff --git a/src/mongo/base/error_codes_test.cpp b/src/mongo/base/error_codes_test.cpp new file mode 100644 index 0000000000..c2298de16b --- /dev/null +++ b/src/mongo/base/error_codes_test.cpp @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2026 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/base/error_codes.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace error_codes_test { + +const bool* namedCodeAddressInOtherTranslationUnit(); +const bool* unnamedCodeAddressInOtherTranslationUnit(); + +namespace { + +TEST(ErrorCodes, NamedCodeVariableTemplatesHaveOneDefinition) { + static_assert(error_details::isNamedCode, "BadValue is named"); + static_assert(!error_details::isNamedCode<-1>, "-1 is not named"); + // Taking addresses verifies the header definitions are shared across translation units, + // even on compilers that otherwise accept non-inline constexpr specializations. + ASSERT_EQ(&error_details::isNamedCode, + namedCodeAddressInOtherTranslationUnit()); + ASSERT_EQ(&error_details::isNamedCode<-1>, unnamedCodeAddressInOtherTranslationUnit()); +} + +} // namespace +} // namespace error_codes_test +} // namespace mongo diff --git a/src/mongo/base/error_codes_test_helper.cpp b/src/mongo/base/error_codes_test_helper.cpp new file mode 100644 index 0000000000..a7e4c46eaa --- /dev/null +++ b/src/mongo/base/error_codes_test_helper.cpp @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2026 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/base/error_codes.h" + +namespace mongo { +namespace error_codes_test { + +const bool* namedCodeAddressInOtherTranslationUnit() { + return &error_details::isNamedCode; +} + +const bool* unnamedCodeAddressInOtherTranslationUnit() { + return &error_details::isNamedCode<-1>; +} + +} // namespace error_codes_test +} // namespace mongo diff --git a/src/mongo/bson/simple_bsonobj_comparator.h b/src/mongo/bson/simple_bsonobj_comparator.h index 59a61e27dd..738adeda59 100644 --- a/src/mongo/bson/simple_bsonobj_comparator.h +++ b/src/mongo/bson/simple_bsonobj_comparator.h @@ -73,7 +73,7 @@ class SimpleBSONObjComparator final : public BSONObj::ComparatorInterface { */ class EqualTo { public: - explicit EqualTo() = default; + EqualTo() = default; bool operator()(const BSONObj& lhs, const BSONObj& rhs) const { return kInstance.compare(lhs, rhs) == 0; @@ -86,7 +86,7 @@ class SimpleBSONObjComparator final : public BSONObj::ComparatorInterface { */ class Hasher { public: - explicit Hasher() = default; + Hasher() = default; size_t operator()(const BSONObj& obj) const { return kInstance.hash(obj); diff --git a/src/mongo/bson/simple_bsonobj_comparator_test.cpp b/src/mongo/bson/simple_bsonobj_comparator_test.cpp index 40d536f0bd..e5e50c513d 100644 --- a/src/mongo/bson/simple_bsonobj_comparator_test.cpp +++ b/src/mongo/bson/simple_bsonobj_comparator_test.cpp @@ -58,6 +58,17 @@ void assertInsertFails(Set& set, BSONObj obj) { << "object was inserted successfully, but should have failed: " << obj.jsonString(); } +TEST(SimpleBSONObjContainerTest, UnorderedFunctorsAreCopyListInitializable) { + // Some standard-library hash tables initialize these stateless functors from an empty list. + SimpleBSONObjComparator::EqualTo equalTo = {}; + SimpleBSONObjComparator::Hasher hasher = {}; + const auto obj = BSON("x" << 1); + const auto same = BSON("x" << 1); + ASSERT_TRUE(equalTo(obj, same)); + ASSERT_FALSE(equalTo(obj, BSON("x" << 2))); + ASSERT_EQ(hasher(obj), hasher(same)); +} + TEST(SimpleBSONObjContainerTest, SetIsDefaultConstructible) { SimpleBSONObjSet set; assertInsertSucceeds(set, BSON("x" << 1)); diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index 9f92e861f5..fe88fe1953 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -1612,6 +1612,7 @@ env.CppUnitTest( ], LIBDEPS=[ 'logical_time', + 'service_context_test_fixture', ], ) diff --git a/src/mongo/db/auth/authorization_manager_test.cpp b/src/mongo/db/auth/authorization_manager_test.cpp index e0ba470d4f..186205f2ba 100644 --- a/src/mongo/db/auth/authorization_manager_test.cpp +++ b/src/mongo/db/auth/authorization_manager_test.cpp @@ -49,6 +49,7 @@ #include "mongo/db/operation_context.h" #include "mongo/db/service_context_test_fixture.h" #include "mongo/db/storage/recovery_unit_noop.h" +#include "mongo/db/storage/storage_engine.h" #include "mongo/stdx/memory.h" #include "mongo/transport/session.h" #include "mongo/transport/transport_layer_mock.h" @@ -74,6 +75,64 @@ void setX509PeerInfo(const transport::SessionHandle& session, SSLPeerInfo info) using std::vector; +// These tests exercise local authorization bookkeeping, not Eloq's cross-node cache +// notifications. Satisfy the storage-engine dependency without starting a real engine. +class AuthorizationStorageEngineMock final : public StorageEngine { +public: + void onAuthzDataChanged(OperationContext*) override {} + + RecoveryUnit* newRecoveryUnit() override { + return new RecoveryUnitNoop(); + } + + bool supportsDocLocking() const override { + return false; + } + + bool isDurable() const override { + return false; + } + + bool isEphemeral() const override { + return true; + } + + void cleanShutdown() override {} + + // No database operations should reach the engine in these mock-backed auth tests. + void listDatabases(std::vector*) const override { + MONGO_UNREACHABLE; + } + + DatabaseCatalogEntry* getDatabaseCatalogEntry(OperationContext*, StringData) override { + MONGO_UNREACHABLE; + } + + Status closeDatabase(OperationContext*, StringData) override { + MONGO_UNREACHABLE; + } + + Status dropDatabase(OperationContext*, StringData) override { + MONGO_UNREACHABLE; + } + + int flushAllFiles(OperationContext*, bool) override { + MONGO_UNREACHABLE; + } + + Status repairRecordStore(OperationContext*, const std::string&) override { + MONGO_UNREACHABLE; + } + + void setJournalListener(JournalListener*) override { + MONGO_UNREACHABLE; + } + + Timestamp getAllCommittedTimestamp() const override { + MONGO_UNREACHABLE; + } +}; + class AuthorizationManagerTest : public ServiceContextTest { public: virtual ~AuthorizationManagerTest() { @@ -82,6 +141,7 @@ class AuthorizationManagerTest : public ServiceContextTest { } AuthorizationManagerTest() { + getServiceContext()->setStorageEngine(std::make_unique()); auto localExternalState = std::make_unique(); externalState = localExternalState.get(); auto localAuthzManager = std::make_unique( diff --git a/src/mongo/db/catalog/collection_mock.h b/src/mongo/db/catalog/collection_mock.h index 2b435027c6..d13c70005c 100644 --- a/src/mongo/db/catalog/collection_mock.h +++ b/src/mongo/db/catalog/collection_mock.h @@ -62,12 +62,16 @@ class CollectionMock : virtual public Collection::Impl, Status recordStoreGoingToUpdateInPlace(OperationContext* opCtx, const RecordId& loc) { std::abort(); } + const std::string _catalogVersion; const NamespaceString _ns; public: const NamespaceString& ns() const { return _ns; } + const std::string& catalogVersion() const { + return _catalogVersion; + } bool ok() const { std::abort(); } @@ -138,6 +142,17 @@ class CollectionMock : virtual public Collection::Impl, std::abort(); } + void deleteDocument(OperationContext* opCtx, + StmtId stmtId, + const RecordId& loc, + const Snapshotted& doc, + OpDebug* opDebug, + bool fromMigrate, + bool noWarn, + Collection::StoreDeletedDoc storeDeletedDoc) { + std::abort(); + } + Status insertDocuments(OperationContext* opCtx, std::vector::const_iterator begin, std::vector::const_iterator end, diff --git a/src/mongo/db/catalog/database_holder_mock.h b/src/mongo/db/catalog/database_holder_mock.h index ab4058e441..9d0d93df06 100644 --- a/src/mongo/db/catalog/database_holder_mock.h +++ b/src/mongo/db/catalog/database_holder_mock.h @@ -51,7 +51,7 @@ class DatabaseHolderMock : public DatabaseHolder::Impl { * Retrieves an already opened database or returns nullptr. Must be called with the database * locked in at least IS-mode. */ - Database* get(OperationContext* opCtx, StringData ns) const override { + Database* get(OperationContext* opCtx, StringData ns) override { return nullptr; } diff --git a/src/mongo/db/catalog/uuid_catalog_test.cpp b/src/mongo/db/catalog/uuid_catalog_test.cpp index 3db1297667..344533d8bb 100644 --- a/src/mongo/db/catalog/uuid_catalog_test.cpp +++ b/src/mongo/db/catalog/uuid_catalog_test.cpp @@ -43,7 +43,6 @@ class UUIDCatalogTest : public unittest::Test { public: UUIDCatalogTest() : nss("testdb", "testcol"), - col(stdx::make_unique(nss)), colUUID(CollectionUUID::gen()), nextUUID(CollectionUUID::gen()), prevUUID(CollectionUUID::gen()) { @@ -57,14 +56,30 @@ class UUIDCatalogTest : public unittest::Test { ASSERT_GT(nextUUID, colUUID); // Register dummy collection in catalog. - catalog.onCreateCollection(&opCtx, &col, colUUID); + col = createCollection(nss, colUUID); + } + + Collection* createCollection(const NamespaceString& collectionNss, CollectionUUID uuid) { + auto owned = stdx::make_unique( + stdx::make_unique(collectionNss)); + auto* collection = owned.get(); + catalog.onCreateCollection(&opCtx, std::move(owned), uuid); + return collection; + } + + Collection* renameCollection(const NamespaceString& collectionNss, CollectionUUID uuid) { + auto owned = stdx::make_unique( + stdx::make_unique(collectionNss)); + auto* collection = owned.get(); + catalog.onRenameCollection(&opCtx, std::move(owned), uuid); + return collection; } protected: UUIDCatalog catalog; OperationContextNoop opCtx; NamespaceString nss; - Collection col; + Collection* col; CollectionUUID colUUID; CollectionUUID nextUUID; CollectionUUID prevUUID; @@ -73,7 +88,7 @@ class UUIDCatalogTest : public unittest::Test { namespace { TEST_F(UUIDCatalogTest, OnCreateCollection) { - ASSERT(catalog.lookupCollectionByUUID(colUUID) == &col); + ASSERT(catalog.lookupCollectionByUUID(colUUID).get() == col); } TEST_F(UUIDCatalogTest, LookupCollectionByUUID) { @@ -94,13 +109,12 @@ TEST_F(UUIDCatalogTest, LookupNSSByUUID) { TEST_F(UUIDCatalogTest, InsertAfterLookup) { auto newUUID = CollectionUUID::gen(); NamespaceString newNss(nss.db(), "newcol"); - Collection newCol(stdx::make_unique(newNss)); // Ensure that looking up non-existing UUIDs doesn't affect later registration of those UUIDs. ASSERT(catalog.lookupCollectionByUUID(newUUID) == nullptr); ASSERT(catalog.lookupNSSByUUID(newUUID) == NamespaceString()); - catalog.onCreateCollection(&opCtx, &newCol, newUUID); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(newUUID), &newCol); + auto* newCol = createCollection(newNss, newUUID); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(newUUID).get(), newCol); ASSERT_EQUALS(catalog.lookupNSSByUUID(colUUID), nss); } @@ -113,14 +127,12 @@ TEST_F(UUIDCatalogTest, OnDropCollection) { TEST_F(UUIDCatalogTest, OnRenameCollection) { auto oldUUID = CollectionUUID::gen(); NamespaceString oldNss(nss.db(), "oldcol"); - Collection oldCol(stdx::make_unique(oldNss)); - catalog.onCreateCollection(&opCtx, &oldCol, oldUUID); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(oldUUID), &oldCol); + auto* oldCol = createCollection(oldNss, oldUUID); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(oldUUID).get(), oldCol); NamespaceString newNss(nss.db(), "newcol"); - Collection newCol(stdx::make_unique(newNss)); - catalog.onRenameCollection(&opCtx, &newCol, oldUUID); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(oldUUID), &newCol); + auto* newCol = renameCollection(newNss, oldUUID); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(oldUUID).get(), newCol); } TEST_F(UUIDCatalogTest, NonExistingNextCol) { @@ -128,20 +140,17 @@ TEST_F(UUIDCatalogTest, NonExistingNextCol) { ASSERT_FALSE(catalog.next(nss.db(), nextUUID)); NamespaceString newNss("anotherdb", "newcol"); - Collection newCol(stdx::make_unique(newNss)); - catalog.onCreateCollection(&opCtx, &newCol, nextUUID); + createCollection(newNss, nextUUID); ASSERT_FALSE(catalog.next(nss.db(), colUUID)); NamespaceString prevNss(nss.db(), "prevcol"); - Collection prevCol(stdx::make_unique(prevNss)); - catalog.onCreateCollection(&opCtx, &prevCol, prevUUID); + createCollection(prevNss, prevUUID); ASSERT_FALSE(catalog.next(nss.db(), colUUID)); } TEST_F(UUIDCatalogTest, ExistingNextCol) { NamespaceString nextNss(nss.db(), "next"); - Collection nextCol(stdx::make_unique(nextNss)); - catalog.onCreateCollection(&opCtx, &nextCol, nextUUID); + createCollection(nextNss, nextUUID); auto next = catalog.next(nss.db(), colUUID); ASSERT_TRUE(next); ASSERT_EQUALS(*next, nextUUID); @@ -152,20 +161,17 @@ TEST_F(UUIDCatalogTest, NonExistingPrevCol) { ASSERT_FALSE(catalog.prev(nss.db(), prevUUID)); NamespaceString newNss("anotherdb", "newcol"); - Collection newCol(stdx::make_unique(newNss)); - catalog.onCreateCollection(&opCtx, &newCol, nextUUID); + createCollection(newNss, nextUUID); ASSERT_FALSE(catalog.prev(nss.db(), colUUID)); NamespaceString nextNss(nss.db(), "nextcol"); - Collection nextCol(stdx::make_unique(nextNss)); - catalog.onCreateCollection(&opCtx, &nextCol, nextUUID); + createCollection(nextNss, nextUUID); ASSERT_FALSE(catalog.prev(nss.db(), colUUID)); } TEST_F(UUIDCatalogTest, ExistingPrevCol) { NamespaceString prevNss(nss.db(), "prevcol"); - Collection prevCol(stdx::make_unique(prevNss)); - catalog.onCreateCollection(&opCtx, &prevCol, prevUUID); + createCollection(prevNss, prevUUID); auto prev = catalog.prev(nss.db(), colUUID); ASSERT_TRUE(prev); ASSERT_EQUALS(*prev, prevUUID); @@ -181,12 +187,10 @@ TEST_F(UUIDCatalogTest, NextPrevColOnEmptyCatalog) { TEST_F(UUIDCatalogTest, InvalidateOrdering) { NamespaceString prevNss(nss.db(), "prevcol"); - Collection prevCol(stdx::make_unique(prevNss)); - catalog.onCreateCollection(&opCtx, &prevCol, prevUUID); + createCollection(prevNss, prevUUID); NamespaceString nextNss(nss.db(), "nextcol"); - Collection nextCol(stdx::make_unique(nextNss)); - catalog.onCreateCollection(&opCtx, &nextCol, nextUUID); + createCollection(nextNss, nextUUID); catalog.onDropCollection(&opCtx, colUUID); @@ -211,36 +215,34 @@ TEST_F(UUIDCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsOldNSSIfDropped) { TEST_F(UUIDCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsNewlyCreatedNSS) { auto newUUID = CollectionUUID::gen(); NamespaceString newNss(nss.db(), "newcol"); - Collection newCol(stdx::make_unique(newNss)); // Ensure that looking up non-existing UUIDs doesn't affect later registration of those UUIDs. catalog.onCloseCatalog(&opCtx); ASSERT(catalog.lookupCollectionByUUID(newUUID) == nullptr); ASSERT(catalog.lookupNSSByUUID(newUUID) == NamespaceString()); - catalog.onCreateCollection(&opCtx, &newCol, newUUID); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(newUUID), &newCol); + auto* newCol = createCollection(newNss, newUUID); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(newUUID).get(), newCol); ASSERT_EQUALS(catalog.lookupNSSByUUID(colUUID), nss); // Ensure that collection still exists after opening the catalog again. catalog.onOpenCatalog(&opCtx); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(newUUID), &newCol); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(newUUID).get(), newCol); ASSERT_EQUALS(catalog.lookupNSSByUUID(colUUID), nss); } TEST_F(UUIDCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsFreshestNSS) { NamespaceString newNss(nss.db(), "newcol"); - Collection newCol(stdx::make_unique(newNss)); catalog.onCloseCatalog(&opCtx); catalog.onDropCollection(&opCtx, colUUID); ASSERT(catalog.lookupCollectionByUUID(colUUID) == nullptr); ASSERT_EQUALS(catalog.lookupNSSByUUID(colUUID), nss); - catalog.onCreateCollection(&opCtx, &newCol, colUUID); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(colUUID), &newCol); + auto* newCol = createCollection(newNss, colUUID); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(colUUID).get(), newCol); ASSERT_EQUALS(catalog.lookupNSSByUUID(colUUID), newNss); // Ensure that collection still exists after opening the catalog again. catalog.onOpenCatalog(&opCtx); - ASSERT_EQUALS(catalog.lookupCollectionByUUID(colUUID), &newCol); + ASSERT_EQUALS(catalog.lookupCollectionByUUID(colUUID).get(), newCol); ASSERT_EQUALS(catalog.lookupNSSByUUID(colUUID), newNss); } } // namespace diff --git a/src/mongo/db/catalog_raii_test.cpp b/src/mongo/db/catalog_raii_test.cpp index 4d178e9d72..07b727e8af 100644 --- a/src/mongo/db/catalog_raii_test.cpp +++ b/src/mongo/db/catalog_raii_test.cpp @@ -37,6 +37,7 @@ #include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/concurrency/lock_state.h" #include "mongo/db/service_context_test_fixture.h" +#include "mongo/db/storage/recovery_unit_noop.h" #include "mongo/unittest/unittest.h" #include "mongo/util/log.h" #include "mongo/util/time_support.h" @@ -52,7 +53,16 @@ class CatalogRAIITestFixture : public ServiceContextTest { ClientAndCtx makeClientWithLocker(const std::string& clientName) { auto client = getServiceContext()->makeClient(clientName); auto opCtx = client->makeOperationContext(); - opCtx->swapLockState(stdx::make_unique()); + // This fixture tests locking without a storage engine. Install its own state + // on both newly allocated and pooled operation contexts. + auto locker = stdx::make_unique(); + if (opCtx->lockState()) { + opCtx->swapLockState(std::move(locker)); + } else { + opCtx->setLockState(std::move(locker)); + } + opCtx->setRecoveryUnit(new RecoveryUnitNoop(), + WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); return std::make_pair(std::move(client), std::move(opCtx)); } diff --git a/src/mongo/db/commands/conn_pool_stats.cpp b/src/mongo/db/commands/conn_pool_stats.cpp index f554b55302..d3411e4617 100644 --- a/src/mongo/db/commands/conn_pool_stats.cpp +++ b/src/mongo/db/commands/conn_pool_stats.cpp @@ -114,6 +114,7 @@ class PoolStats final : public BasicCommand { } poolStatsCmd; +#ifndef ELOQDOC_STANDALONE class ShardedPoolStats final : public BasicCommand { public: ShardedPoolStats() : BasicCommand("shardConnPoolStats") {} @@ -156,6 +157,7 @@ class ShardedPoolStats final : public BasicCommand { } } shardedPoolStatsCmd; +#endif } // namespace } // namespace mongo diff --git a/src/mongo/db/commands/conn_pool_sync.cpp b/src/mongo/db/commands/conn_pool_sync.cpp index 31b9291357..7fbc91e9e6 100644 --- a/src/mongo/db/commands/conn_pool_sync.cpp +++ b/src/mongo/db/commands/conn_pool_sync.cpp @@ -63,7 +63,9 @@ class PoolFlushCmd : public BasicCommand { const std::string&, const mongo::BSONObj&, mongo::BSONObjBuilder& result) override { +#ifndef ELOQDOC_STANDALONE shardConnectionPool.flush(); +#endif globalConnPool.flush(); return true; } diff --git a/src/mongo/db/commands/fsync.cpp b/src/mongo/db/commands/fsync.cpp index 81807c7e52..13074b739f 100644 --- a/src/mongo/db/commands/fsync.cpp +++ b/src/mongo/db/commands/fsync.cpp @@ -48,7 +48,6 @@ #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db.h" #include "mongo/db/service_context.h" -#include "mongo/db/storage/mmap_v1/dur.h" #include "mongo/db/storage/storage_engine.h" #include "mongo/stdx/condition_variable.h" #include "mongo/util/assert_util.h" @@ -136,16 +135,6 @@ class FSyncCommand : public ErrmsgCommandDeprecated { if (!lock) { // the simple fsync command case - if (sync) { - // can this be GlobalRead? and if it can, it should be nongreedy. - Lock::GlobalWrite w(opCtx); - // TODO SERVER-26822: Replace MMAPv1 specific calls with ones that are storage - // engine agnostic. - getDur().commitNow(opCtx); - - // No WriteUnitOfWork needed, as this does no writes of its own. - } - // Take a global IS lock to ensure the storage engine is not shutdown Lock::GlobalLock global(opCtx, MODE_IS); StorageEngine* storageEngine = getGlobalServiceContext()->getStorageEngine(); @@ -346,16 +335,6 @@ void FSyncLockThread::run() { OperationContext& opCtx = *opCtxPtr; Lock::GlobalWrite global(&opCtx); // No WriteUnitOfWork needed - try { - // TODO SERVER-26822: Replace MMAPv1 specific calls with ones that are storage engine - // agnostic. - getDur().syncDataAndTruncateJournal(&opCtx); - } catch (const std::exception& e) { - error() << "error doing syncDataAndTruncateJournal: " << e.what(); - fsyncCmd.threadStatus = Status(ErrorCodes::CommandFailed, e.what()); - fsyncCmd.acquireFsyncLockSyncCV.notify_one(); - return; - } opCtx.lockState()->downgradeGlobalXtoSForMMAPV1(); StorageEngine* storageEngine = getGlobalServiceContext()->getStorageEngine(); diff --git a/src/mongo/db/commands/index_filter_commands_test.cpp b/src/mongo/db/commands/index_filter_commands_test.cpp index 7ee3a62e25..9a42e852e7 100644 --- a/src/mongo/db/commands/index_filter_commands_test.cpp +++ b/src/mongo/db/commands/index_filter_commands_test.cpp @@ -126,14 +126,14 @@ void addQueryShapeToPlanCache(OperationContext* opCtx, const char* projectionStr, const char* collationStr) { // Create canonical query. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projectionStr)); qr->setCollation(fromjson(collationStr)); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx, std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - std::unique_ptr cq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); QuerySolution qs; qs.cacheData.reset(new SolutionCacheData()); @@ -157,14 +157,14 @@ bool planCacheContains(OperationContext* opCtx, const char* collationStr) { // Create canonical query. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projectionStr)); qr->setCollation(fromjson(collationStr)); auto statusWithInputQuery = CanonicalQuery::canonicalize(opCtx, std::move(qr)); ASSERT_OK(statusWithInputQuery.getStatus()); - unique_ptr inputQuery = std::move(statusWithInputQuery.getValue()); + CanonicalQuery::UPtr inputQuery = std::move(statusWithInputQuery.getValue()); // Retrieve cache entries from plan cache. vector entries = planCache.getAllEntries(); @@ -177,14 +177,14 @@ bool planCacheContains(OperationContext* opCtx, // Canonicalizing query shape in cache entry to get cache key. // Alternatively, we could add key to PlanCacheEntry but that would be used in one place // only. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(entry->query); qr->setSort(entry->sort); qr->setProj(entry->projection); qr->setCollation(entry->collation); auto statusWithCurrentQuery = CanonicalQuery::canonicalize(opCtx, std::move(qr)); ASSERT_OK(statusWithCurrentQuery.getStatus()); - unique_ptr currentQuery = std::move(statusWithCurrentQuery.getValue()); + CanonicalQuery::UPtr currentQuery = std::move(statusWithCurrentQuery.getValue()); if (planCache.computeKey(*currentQuery) == planCache.computeKey(*inputQuery)) { found = true; diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp index e798ca0c9a..ddbeb70008 100644 --- a/src/mongo/db/commands/mr.cpp +++ b/src/mongo/db/commands/mr.cpp @@ -392,7 +392,9 @@ void State::dropTempCollections() { } }); // Always forget about temporary namespaces, so we don't cache lots of them +#ifndef ELOQDOC_STANDALONE ShardConnection::forgetNS(_config.tempNamespace.ns()); +#endif } if (_useIncremental && !_config.incLong.isEmpty()) { writeConflictRetry(_opCtx, "M/R dropTempCollections", _config.incLong.ns(), [this] { @@ -405,7 +407,9 @@ void State::dropTempCollections() { } }); +#ifndef ELOQDOC_STANDALONE ShardConnection::forgetNS(_config.incLong.ns()); +#endif } } @@ -1694,6 +1698,7 @@ class MapReduceCommand : public ErrmsgCommandDeprecated { /** * This class represents a map/reduce command executed on the output server of a sharded env */ +#ifndef ELOQDOC_STANDALONE class MapReduceFinishCommand : public BasicCommand { public: std::string help() const override { @@ -1893,6 +1898,7 @@ class MapReduceFinishCommand : public BasicCommand { } } mapReduceFinishCommand; +#endif } // namespace mr } // namespace mongo diff --git a/src/mongo/db/commands/plan_cache_commands_test.cpp b/src/mongo/db/commands/plan_cache_commands_test.cpp index 43d0f7d383..f80eb01614 100644 --- a/src/mongo/db/commands/plan_cache_commands_test.cpp +++ b/src/mongo/db/commands/plan_cache_commands_test.cpp @@ -136,14 +136,14 @@ TEST(PlanCacheCommandsTest, planCacheListQueryShapesOneKey) { auto opCtx = serviceContext.makeOperationContext(); // Create a canonical query - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 1}")); qr->setSort(fromjson("{a: -1}")); qr->setProj(fromjson("{_id: 0}")); qr->setCollation(fromjson("{locale: 'mock_reverse_string'}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr cq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); // Plan cache with one entry PlanCache planCache; @@ -173,11 +173,11 @@ TEST(PlanCacheCommandsTest, planCacheClearAllShapes) { auto opCtx = serviceContext.makeOperationContext(); // Create a canonical query - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 1}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr cq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); // Plan cache with one entry PlanCache planCache; @@ -235,41 +235,41 @@ TEST(PlanCacheCommandsTest, Canonicalize) { auto statusWithCQ = PlanCacheCommand::canonicalize(opCtx.get(), nss.ns(), fromjson("{query: {a: 1, b: 1}}")); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr query = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr query = std::move(statusWithCQ.getValue()); // Equivalent query should generate same key. statusWithCQ = PlanCacheCommand::canonicalize(opCtx.get(), nss.ns(), fromjson("{query: {b: 1, a: 1}}")); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr equivQuery = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr equivQuery = std::move(statusWithCQ.getValue()); ASSERT_EQUALS(planCache.computeKey(*query), planCache.computeKey(*equivQuery)); // Sort query should generate different key from unsorted query. statusWithCQ = PlanCacheCommand::canonicalize( opCtx.get(), nss.ns(), fromjson("{query: {a: 1, b: 1}, sort: {a: 1, b: 1}}")); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr sortQuery1 = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr sortQuery1 = std::move(statusWithCQ.getValue()); ASSERT_NOT_EQUALS(planCache.computeKey(*query), planCache.computeKey(*sortQuery1)); // Confirm sort arguments are properly delimited (SERVER-17158) statusWithCQ = PlanCacheCommand::canonicalize( opCtx.get(), nss.ns(), fromjson("{query: {a: 1, b: 1}, sort: {aab: 1}}")); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr sortQuery2 = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr sortQuery2 = std::move(statusWithCQ.getValue()); ASSERT_NOT_EQUALS(planCache.computeKey(*sortQuery1), planCache.computeKey(*sortQuery2)); // Changing order and/or value of predicates should not change key statusWithCQ = PlanCacheCommand::canonicalize( opCtx.get(), nss.ns(), fromjson("{query: {b: 3, a: 3}, sort: {a: 1, b: 1}}")); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr sortQuery3 = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr sortQuery3 = std::move(statusWithCQ.getValue()); ASSERT_EQUALS(planCache.computeKey(*sortQuery1), planCache.computeKey(*sortQuery3)); // Projected query should generate different key from unprojected query. statusWithCQ = PlanCacheCommand::canonicalize( opCtx.get(), nss.ns(), fromjson("{query: {a: 1, b: 1}, projection: {_id: 0, a: 1}}")); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr projectionQuery = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr projectionQuery = std::move(statusWithCQ.getValue()); ASSERT_NOT_EQUALS(planCache.computeKey(*query), planCache.computeKey(*projectionQuery)); } @@ -310,16 +310,16 @@ TEST(PlanCacheCommandsTest, planCacheClearOneKey) { auto opCtx = serviceContext.makeOperationContext(); // Create 2 canonical queries. - auto qrA = stdx::make_unique(nss); + auto qrA = ObjectPool::newObject(nss); qrA->setFilter(fromjson("{a: 1}")); auto statusWithCQA = CanonicalQuery::canonicalize(opCtx.get(), std::move(qrA)); ASSERT_OK(statusWithCQA.getStatus()); - auto qrB = stdx::make_unique(nss); + auto qrB = ObjectPool::newObject(nss); qrB->setFilter(fromjson("{b: 1}")); - unique_ptr cqA = std::move(statusWithCQA.getValue()); + CanonicalQuery::UPtr cqA = std::move(statusWithCQA.getValue()); auto statusWithCQB = CanonicalQuery::canonicalize(opCtx.get(), std::move(qrB)); ASSERT_OK(statusWithCQB.getStatus()); - unique_ptr cqB = std::move(statusWithCQB.getValue()); + CanonicalQuery::UPtr cqB = std::move(statusWithCQB.getValue()); // Create plan cache with 2 entries. PlanCache planCache; @@ -369,17 +369,17 @@ TEST(PlanCacheCommandsTest, planCacheClearOneKeyCollation) { auto opCtx = serviceContext.makeOperationContext(); // Create 2 canonical queries, one with collation. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 'foo'}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr cq = std::move(statusWithCQ.getValue()); - auto qrCollation = stdx::make_unique(nss); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); + auto qrCollation = ObjectPool::newObject(nss); qrCollation->setFilter(fromjson("{a: 'foo'}")); qrCollation->setCollation(fromjson("{locale: 'mock_reverse_string'}")); auto statusWithCQCollation = CanonicalQuery::canonicalize(opCtx.get(), std::move(qrCollation)); ASSERT_OK(statusWithCQCollation.getStatus()); - unique_ptr cqCollation = std::move(statusWithCQCollation.getValue()); + CanonicalQuery::UPtr cqCollation = std::move(statusWithCQCollation.getValue()); // Create plan cache with 2 entries. Add an index so that indexability is included in the plan // cache keys. @@ -527,11 +527,11 @@ TEST(PlanCacheCommandsTest, planCacheListPlansOnlyOneSolutionTrue) { auto opCtx = serviceContext.makeOperationContext(); // Create a canonical query - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 1}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr cq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); // Plan cache with one entry PlanCache planCache; @@ -557,11 +557,11 @@ TEST(PlanCacheCommandsTest, planCacheListPlansOnlyOneSolutionFalse) { auto opCtx = serviceContext.makeOperationContext(); // Create a canonical query - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 1}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr cq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); // Plan cache with one entry PlanCache planCache; @@ -590,17 +590,17 @@ TEST(PlanCacheCommandsTest, planCacheListPlansCollation) { auto opCtx = serviceContext.makeOperationContext(); // Create 2 canonical queries, one with collation. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 'foo'}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr cq = std::move(statusWithCQ.getValue()); - auto qrCollation = stdx::make_unique(nss); + CanonicalQuery::UPtr cq = std::move(statusWithCQ.getValue()); + auto qrCollation = ObjectPool::newObject(nss); qrCollation->setFilter(fromjson("{a: 'foo'}")); qrCollation->setCollation(fromjson("{locale: 'mock_reverse_string'}")); auto statusWithCQCollation = CanonicalQuery::canonicalize(opCtx.get(), std::move(qrCollation)); ASSERT_OK(statusWithCQCollation.getStatus()); - unique_ptr cqCollation = std::move(statusWithCQCollation.getValue()); + CanonicalQuery::UPtr cqCollation = std::move(statusWithCQCollation.getValue()); // Create plan cache with 2 entries. Add an index so that indexability is included in the plan // cache keys. Give query with collation two solutions. @@ -645,7 +645,7 @@ TEST(PlanCacheCommandsTest, planCacheListPlansTimeOfCreationIsCorrect) { auto opCtx = serviceContext.makeOperationContext(); // Create a canonical query. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 1}")); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); diff --git a/src/mongo/db/commands/restart_catalog_command.cpp b/src/mongo/db/commands/restart_catalog_command.cpp index d7958defdb..895912590c 100644 --- a/src/mongo/db/commands/restart_catalog_command.cpp +++ b/src/mongo/db/commands/restart_catalog_command.cpp @@ -39,6 +39,7 @@ #include "mongo/db/commands.h" #include "mongo/db/commands/test_commands_enabled.h" #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/storage/storage_options.h" #include "mongo/util/log.h" namespace mongo { @@ -87,6 +88,11 @@ class RestartCatalogCmd final : public BasicCommand { const std::string& db, const BSONObj& cmdObj, BSONObjBuilder& result) final { + // This command depends on a real global exclusive lock to protect every worker's + // catalog pointers. Eloq's no-op locker cannot provide that stop-the-world guarantee. + uassert(ErrorCodes::CommandNotSupported, + "Eloq storage engine does not support online catalog restart", + storageGlobalParams.engine != "eloq"); Lock::GlobalLock global(opCtx, MODE_X); // This command will fail without modifying the catalog if there are any databases that are diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp index 1d6ca50d71..5744da5ddb 100644 --- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp +++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp @@ -172,6 +172,7 @@ class SetFeatureCompatibilityVersionCommand : public BasicCommand { updateUniqueIndexesOnUpgrade(opCtx); +#ifndef ELOQDOC_STANDALONE // Upgrade shards before config finishes its upgrade. if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) { auto allDbs = uassertStatusOK(Grid::get(opCtx)->catalogClient()->getAllDBs( @@ -224,6 +225,7 @@ class SetFeatureCompatibilityVersionCommand : public BasicCommand { << requestedVersion))))); } +#endif FeatureCompatibilityVersion::unsetTargetUpgradeOrDowngrade(opCtx, requestedVersion); } else if (requestedVersion == FeatureCompatibilityVersionParser::kVersion36) { uassert(ErrorCodes::IllegalOperation, @@ -254,6 +256,7 @@ class SetFeatureCompatibilityVersionCommand : public BasicCommand { Lock::GlobalLock lk(opCtx, MODE_S); } +#ifndef ELOQDOC_STANDALONE // Downgrade shards before config finishes its downgrade. if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) { uassertStatusOK( @@ -303,6 +306,7 @@ class SetFeatureCompatibilityVersionCommand : public BasicCommand { Grid::get(opCtx)->catalogCache()->purgeAllDatabases(); } +#endif FeatureCompatibilityVersion::unsetTargetUpgradeOrDowngrade(opCtx, requestedVersion); } diff --git a/src/mongo/db/concurrency/d_concurrency_test.cpp b/src/mongo/db/concurrency/d_concurrency_test.cpp index 2fa499e2f0..18613c98c9 100644 --- a/src/mongo/db/concurrency/d_concurrency_test.cpp +++ b/src/mongo/db/concurrency/d_concurrency_test.cpp @@ -35,6 +35,7 @@ #include #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/eloq_locker_noop.h" #include "mongo/db/concurrency/global_lock_acquisition_tracker.h" #include "mongo/db/concurrency/lock_manager_test_help.h" #include "mongo/db/concurrency/write_conflict_exception.h" @@ -123,6 +124,43 @@ class DConcurrencyTestFixture : public ServiceContextMongoDTest { }; +TEST_F(DConcurrencyTestFixture, FixtureInstallsDefaultLockerAcrossPoolReuse) { + // Use Client directly so the fixture must cover every operation-context checkout, + // not just calls to the fixture's makeOperationContext() helper. + auto client = getServiceContext()->makeClient("fixture_locker_reuse"); + for (int i = 0; i < 64; ++i) { + auto opCtx = client->makeOperationContext(); + ASSERT_TRUE(dynamic_cast(opCtx->lockState())); + ASSERT_TRUE(opCtx->recoveryUnit()); + Lock::GlobalLock lock(opCtx.get(), MODE_IX); + ASSERT_TRUE(opCtx->lockState()->isLocked()); + } +} + +TEST_F(DConcurrencyTestFixture, FixtureLockerDoesNotLeakIntoBareServiceContext) { + // Populate the thread-local pool with fixture contexts using real MongoDB lockers. + for (int i = 0; i < 64; ++i) { + auto opCtx = makeOperationContext(); + ASSERT_TRUE(dynamic_cast(opCtx->lockState())); + } + + auto bareService = ServiceContext::make(); + auto bareClient = bareService->makeClient("bare_locker_reuse"); + ASSERT_FALSE(bareService->getStorageEngine()); + for (int i = 0; i < 64; ++i) { + auto opCtx = bareClient->makeOperationContext(); + ASSERT_TRUE(dynamic_cast(opCtx->lockState())); + ASSERT_FALSE(opCtx->recoveryUnit()); + } + + // Returning to the fixture must reinstall its real locker after production reuse. + for (int i = 0; i < 64; ++i) { + auto opCtx = makeOperationContext(); + ASSERT_TRUE(dynamic_cast(opCtx->lockState())); + ASSERT_TRUE(opCtx->recoveryUnit()); + } +} + TEST_F(DConcurrencyTestFixture, WriteConflictRetryInstantiatesOK) { auto opCtx = makeOperationContext(); opCtx->swapLockState(stdx::make_unique()); diff --git a/src/mongo/db/coro_sync.cpp b/src/mongo/db/coro_sync.cpp index dc6d903b90..8c6039cbd8 100644 --- a/src/mongo/db/coro_sync.cpp +++ b/src/mongo/db/coro_sync.cpp @@ -42,31 +42,22 @@ void Mutex::lock() { void ConditionVariable::wait(std::unique_lock& lock) { invariant(lock.owns_lock()); - if (LocalThread::ID() != -1) { - Client* client = Client::getCurrent(); - if (client) { - const CoroutineFunctors& coro = Client::getCurrent()->coroutineFunctors(); - if (coro != CoroutineFunctors::Unavailable) { - lock.unlock(); - (*coro.longResumeFuncPtr)(); - (*coro.yieldFuncPtr)(); - lock.lock(); - } else { - MONGO_LOG(1) << "ThreadGroup " << LocalThread::ID() - << " call std::condition_variable::wait because the coroutine context " - "is unavailable."; - _cv.wait(reinterpret_cast&>(lock)); - } - } else { - MONGO_LOG(1) - << "ThreadGroup " << LocalThread::ID() - << " call std::condition_variable::wait because the client object is unavailable."; - _cv.wait(reinterpret_cast&>(lock)); - } - - } else { + if (!_yieldIfCoroutine(lock)) { _cv.wait(reinterpret_cast&>(lock)); } } + +bool ConditionVariable::_yieldIfCoroutine(std::unique_lock& lock) { + if (LocalThread::ID() == -1 || !Client::getCurrent()) + return false; + const auto& coro = Client::getCurrent()->coroutineFunctors(); + if (coro == CoroutineFunctors::Unavailable) + return false; + lock.unlock(); + (*coro.longResumeFuncPtr)(); + (*coro.yieldFuncPtr)(); + lock.lock(); + return true; +} } // namespace coro } // namespace mongo diff --git a/src/mongo/db/coro_sync.h b/src/mongo/db/coro_sync.h index 6c5cb78f9c..ce6d6f98e7 100644 --- a/src/mongo/db/coro_sync.h +++ b/src/mongo/db/coro_sync.h @@ -74,8 +74,14 @@ class ConditionVariable { template std::cv_status wait_until(std::unique_lock& lock, const std::chrono::time_point& timeout_time) { - wait(lock); - return Clock::now() < timeout_time ? std::cv_status::no_timeout : std::cv_status::timeout; + if (Clock::now() >= timeout_time) + return std::cv_status::timeout; + if (_yieldIfCoroutine(lock)) { + return Clock::now() < timeout_time ? std::cv_status::no_timeout + : std::cv_status::timeout; + } + return _cv.wait_until(reinterpret_cast&>(lock), + timeout_time); } template @@ -105,11 +111,11 @@ class ConditionVariable { } std::cv_status wait_until(std::unique_lock& lock, Date_t timeout_time) { - wait(lock); - return Date_t::now() < timeout_time ? std::cv_status::no_timeout : std::cv_status::timeout; + return wait_until(lock, timeout_time.toSystemTimePoint()); } private: + bool _yieldIfCoroutine(std::unique_lock& lock); std::condition_variable _cv; }; } // namespace coro diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 7f148932ab..a69e000e12 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -92,6 +92,7 @@ #include "mongo/db/logical_time_validator.h" #include "mongo/db/mongod_options.h" #include "mongo/db/op_observer_registry.h" +#include "mongo/db/op_observer_impl.h" #include "mongo/db/operation_context.h" #include "mongo/db/periodic_runner_job_abort_expired_transactions.h" #include "mongo/db/query/internal_plans.h" @@ -99,14 +100,19 @@ #include "mongo/db/repl/drop_pending_collection_reaper.h" #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/repl_settings.h" -#include "mongo/db/repl/replication_consistency_markers_impl.h" #include "mongo/db/repl/replication_coordinator.h" +#ifdef ELOQDOC_STANDALONE +#include "mongo/db/repl/replication_coordinator_standalone.h" +#else +#include "mongo/db/repl/replication_consistency_markers_impl.h" #include "mongo/db/repl/replication_coordinator_external_state_impl.h" #include "mongo/db/repl/replication_coordinator_impl.h" -#include "mongo/db/repl/replication_process.h" #include "mongo/db/repl/replication_recovery.h" -#include "mongo/db/repl/storage_interface_impl.h" #include "mongo/db/repl/topology_coordinator.h" +#endif +#include "mongo/db/repl/replication_process.h" +#include "mongo/db/repl/storage_interface_impl.h" +#ifndef ELOQDOC_STANDALONE #include "mongo/db/s/balancer/balancer.h" #include "mongo/db/s/config/sharding_catalog_manager.h" #include "mongo/db/s/config_server_op_observer.h" @@ -114,6 +120,7 @@ #include "mongo/db/s/shard_server_op_observer.h" #include "mongo/db/s/sharding_initialization_mongod.h" #include "mongo/db/s/sharding_state_recovery.h" +#endif #include "mongo/db/server_options.h" #include "mongo/db/server_parameters.h" #include "mongo/db/service_context.h" @@ -286,11 +293,13 @@ ExitCode _initAndListen(int listenPort) { opObserverRegistry->addObserver(stdx::make_unique()); +#ifndef ELOQDOC_STANDALONE if (serverGlobalParams.clusterRole == ClusterRole::ShardServer) { opObserverRegistry->addObserver(stdx::make_unique()); } else if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) { opObserverRegistry->addObserver(stdx::make_unique()); } +#endif setupFreeMonitoringOpObserver(opObserverRegistry.get()); @@ -525,12 +534,14 @@ ExitCode _initAndListen(int listenPort) { << startupWarningsLog; } +#ifndef ELOQDOC_STANDALONE // This function may take the global lock. auto shardingInitialized = ShardingInitializationMongoD::get(startupOpCtx.get()) ->initializeShardingAwarenessIfNeeded(startupOpCtx.get()); if (shardingInitialized) { waitForShardRegistryReload(startupOpCtx.get()).transitional_ignore(); } +#endif auto storageEngine = serviceContext->getStorageEngine(); invariant(storageEngine); @@ -547,6 +558,7 @@ ExitCode _initAndListen(int listenPort) { restartInProgressIndexesFromLastShutdown(startupOpCtx.get()); +#ifndef ELOQDOC_STANDALONE if (serverGlobalParams.clusterRole == ClusterRole::ShardServer) { // Note: For replica sets, ShardingStateRecovery happens on transition to primary. if (!repl::ReplicationCoordinator::get(startupOpCtx.get())->isReplEnabled()) { @@ -573,6 +585,7 @@ ExitCode _initAndListen(int listenPort) { LogicalTimeValidator::set(startupOpCtx->getServiceContext(), stdx::make_unique(keyManager)); } +#endif repl::ReplicationCoordinator::get(startupOpCtx.get())->startup(startupOpCtx.get()); const unsigned long long missingRepl = @@ -801,6 +814,7 @@ void startupConfigActions(const std::vector& args) { #endif } +#ifndef ELOQDOC_STANDALONE auto makeReplicationExecutor(ServiceContext* serviceContext) { ThreadPool::Options tpOptions; tpOptions.poolName = "replexec"; @@ -814,9 +828,23 @@ auto makeReplicationExecutor(ServiceContext* serviceContext) { stdx::make_unique(tpOptions), executor::makeNetworkInterface("Replication", nullptr, std::move(hookList))); } +#endif void setUpReplication(ServiceContext* serviceContext) { repl::StorageInterface::set(serviceContext, stdx::make_unique()); +#ifdef ELOQDOC_STANDALONE + // Preserve the server startup path, but omit MongoDB replica-set services. Eloq's + // transaction and durability services are initialized by the storage engine. + // Unsupported topology options are rejected by storeMongodOptions before startup. + // ReplicationProcess and DropPendingCollectionReaper are deliberately not installed. + // Callers must check replication mode before accessing either service; keep this + // contract when merging upstream startup, catalog, and shutdown changes. + invariant(!getGlobalReplSettings().usingReplSets() && + serverGlobalParams.clusterRole == ClusterRole::None); + LogicalClock::set(serviceContext, stdx::make_unique(serviceContext)); + repl::ReplicationCoordinator::set( + serviceContext, stdx::make_unique(serviceContext)); +#else auto storageInterface = repl::StorageInterface::get(serviceContext); auto consistencyMarkers = @@ -851,6 +879,7 @@ void setUpReplication(ServiceContext* serviceContext) { storageInterface, static_cast(curTimeMillis64())); repl::ReplicationCoordinator::set(serviceContext, std::move(replCoord)); +#endif repl::setOplogCollectionName(serviceContext); } @@ -895,7 +924,9 @@ void shutdownTask() { // is building an index. repl::ReplicationCoordinator::get(serviceContext)->shutdown(opCtx); +#ifndef ELOQDOC_STANDALONE ShardingInitializationMongoD::get(serviceContext)->shutDown(opCtx); +#endif // Destroy all stashed transaction resources, in order to release locks. SessionKiller::Matcher matcherAllSessions( @@ -905,6 +936,10 @@ void shutdownTask() { serviceContext->setKillAllOperations(); + // TTL is a separate BackgroundJob, not part of the periodic runner. Drain it while + // Data Substrate and its workers are alive, before taking the global shutdown lock. + shutdownTTLBackgroundJob(); + // Shut down the background periodic task runner if (auto runner = serviceContext->getPeriodicRunner()) { runner->shutdown(); @@ -912,9 +947,11 @@ void shutdownTask() { ReplicaSetMonitor::shutdown(); +#ifndef ELOQDOC_STANDALONE if (auto sr = Grid::get(serviceContext)->shardRegistry()) { sr->shutdown(); } +#endif // Validator shutdown must be called after setKillAllOperations is called. Otherwise, this can // deadlock. diff --git a/src/mongo/db/ftdc/ftdc_test.cpp b/src/mongo/db/ftdc/ftdc_test.cpp index 631ca23397..4cdeb80ed8 100644 --- a/src/mongo/db/ftdc/ftdc_test.cpp +++ b/src/mongo/db/ftdc/ftdc_test.cpp @@ -36,9 +36,11 @@ #include "mongo/base/init.h" #include "mongo/bson/bson_validate.h" #include "mongo/db/client.h" +#include "mongo/db/concurrency/locker_noop.h" #include "mongo/db/ftdc/file_reader.h" #include "mongo/db/jsobj.h" #include "mongo/db/service_context.h" +#include "mongo/db/storage/recovery_unit_noop.h" #include "mongo/stdx/memory.h" #include "mongo/unittest/unittest.h" #include "mongo/util/clock_source.h" @@ -49,6 +51,24 @@ namespace mongo { namespace { +// FTDC's background collector creates operation contexts directly through its Client. +// Install mock storage state for this engine-less fixture, including after pool reuse. +class FTDCStorageObserver final : public ServiceContext::ClientObserver { +public: + void onCreateClient(Client*) override {} + void onDestroyClient(Client*) override {} + void onCreateOperationContext(OperationContext* opCtx) override { + if (!opCtx->lockState()) { + opCtx->setLockState(stdx::make_unique()); + } + if (!opCtx->recoveryUnit()) { + opCtx->setRecoveryUnit(new RecoveryUnitNoop(), + WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); + } + } + void onDestroyOperationContext(OperationContext*) override {} +}; + BSONObj filteredFTDCCopy(const BSONObj& obj) { BSONObjBuilder builder; for (const auto& f : obj) { @@ -138,11 +158,20 @@ void createDirectoryClean(const boost::filesystem::path& dir) { boost::filesystem::create_directory(dir); } -FTDCTest::FTDCTest() { +FTDCTest::FTDCTest() : ServiceContextTest(stdx::make_unique()) { auto service = getServiceContext(); service->setFastClockSource(stdx::make_unique()); service->setPreciseClockSource(stdx::make_unique()); service->setTickSource(stdx::make_unique()); } +TEST_F(FTDCTest, InstallsNoopStorageStateAcrossPoolReuse) { + ASSERT_FALSE(getServiceContext()->getStorageEngine()); + for (int i = 0; i < 64; ++i) { + auto opCtx = makeOperationContext(); + ASSERT_TRUE(dynamic_cast(opCtx->lockState())); + ASSERT_TRUE(dynamic_cast(opCtx->recoveryUnit())); + } +} + } // namespace mongo diff --git a/src/mongo/db/logical_session_cache_factory_standalone.cpp b/src/mongo/db/logical_session_cache_factory_standalone.cpp new file mode 100644 index 0000000000..7af0e88b81 --- /dev/null +++ b/src/mongo/db/logical_session_cache_factory_standalone.cpp @@ -0,0 +1,25 @@ +// CMake-only implementation of the normal mongod logical-session-cache factory. +// The SCons build continues to use logical_session_cache_factory_mongod.cpp. + +#include "mongo/platform/basic.h" + +#include "mongo/db/logical_session_cache_factory_mongod.h" + +#include "mongo/db/logical_session_cache_impl.h" +#include "mongo/db/service_liaison_mongod.h" +#include "mongo/db/sessions_collection_standalone.h" +#include "mongo/stdx/memory.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + +std::unique_ptr makeLogicalSessionCacheD(LogicalSessionCacheServer state) { + invariant(state == LogicalSessionCacheServer::kStandalone); + return stdx::make_unique( + stdx::make_unique(), + std::make_shared(), + nullptr, // MongoDB's transaction reaper is only used by replica sets and shards. + LogicalSessionCacheImpl::Options{}); +} + +} // namespace mongo diff --git a/src/mongo/db/logical_session_id_test.cpp b/src/mongo/db/logical_session_id_test.cpp index e32e1e8808..08dcc17045 100644 --- a/src/mongo/db/logical_session_id_test.cpp +++ b/src/mongo/db/logical_session_id_test.cpp @@ -292,21 +292,22 @@ TEST_F(LogicalSessionIdTest, InitializeOperationSessionInfo_SessionIdAndTransact ASSERT_EQ(100, *_opCtx->getTxnNumber()); } -TEST_F(LogicalSessionIdTest, InitializeOperationSessionInfo_IsReplSetMemberOrMongosFalse) { +TEST_F(LogicalSessionIdTest, InitializeOperationSessionInfo_StandaloneSupportsTransactionNumber) { addSimpleUser(UserName("simple", "test")); LogicalSessionFromClient lsid; lsid.setId(UUID::gen()); - ASSERT_THROWS_CODE( - initializeOperationSessionInfo( - _opCtx.get(), - BSON("TestCmd" << 1 << "lsid" << lsid.toBSON() << "txnNumber" << 100LL << "OtherField" - << "TestField"), - true, - false, - true), - AssertionException, - ErrorCodes::IllegalOperation); + // Eloq provides transactions through its storage engine without MongoDB replication/sharding. + initializeOperationSessionInfo( + _opCtx.get(), + BSON("TestCmd" << 1 << "lsid" << lsid.toBSON() << "txnNumber" << 100LL), + true, + false, + true); + ASSERT(_opCtx->getLogicalSessionId()); + ASSERT_EQ(lsid.getId(), _opCtx->getLogicalSessionId()->getId()); + ASSERT(_opCtx->getTxnNumber()); + ASSERT_EQ(100, *_opCtx->getTxnNumber()); } TEST_F(LogicalSessionIdTest, InitializeOperationSessionInfo_SupportsDocLockingFalse) { diff --git a/src/mongo/db/matcher/expression_optimize_test.cpp b/src/mongo/db/matcher/expression_optimize_test.cpp index 036a1146b2..4327fb6706 100644 --- a/src/mongo/db/matcher/expression_optimize_test.cpp +++ b/src/mongo/db/matcher/expression_optimize_test.cpp @@ -77,7 +77,7 @@ Status isValid(const std::string& queryStr, const QueryRequest& qrRaw) { TEST(ExpressionOptimizeTest, IsValidText) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); ASSERT_OK(qr->validate()); // Valid: regular TEXT. @@ -133,7 +133,7 @@ TEST(ExpressionOptimizeTest, IsValidText) { TEST(ExpressionOptimizeTest, IsValidTextTailable) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setTailableMode(TailableModeEnum::kTailable); ASSERT_OK(qr->validate()); @@ -143,7 +143,7 @@ TEST(ExpressionOptimizeTest, IsValidTextTailable) { TEST(ExpressionOptimizeTest, IsValidGeo) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); ASSERT_OK(qr->validate()); // Valid: regular GEO_NEAR. @@ -209,7 +209,7 @@ TEST(ExpressionOptimizeTest, IsValidGeo) { TEST(ExpressionOptimizeTest, IsValidTextAndGeo) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); ASSERT_OK(qr->validate()); // Invalid: TEXT and GEO_NEAR. @@ -230,7 +230,7 @@ TEST(ExpressionOptimizeTest, IsValidTextAndGeo) { TEST(ExpressionOptimizeTest, IsValidTextAndNaturalAscending) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setSort(fromjson("{$natural: 1}")); ASSERT_OK(qr->validate()); @@ -240,7 +240,7 @@ TEST(ExpressionOptimizeTest, IsValidTextAndNaturalAscending) { TEST(ExpressionOptimizeTest, IsValidTextAndNaturalDescending) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setSort(fromjson("{$natural: -1}")); ASSERT_OK(qr->validate()); @@ -250,7 +250,7 @@ TEST(ExpressionOptimizeTest, IsValidTextAndNaturalDescending) { TEST(ExpressionOptimizeTest, IsValidTextAndHint) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setHint(fromjson("{a: 1}")); ASSERT_OK(qr->validate()); @@ -261,7 +261,7 @@ TEST(ExpressionOptimizeTest, IsValidTextAndHint) { // SERVER-14366 TEST(ExpressionOptimizeTest, IsValidGeoNearNaturalSort) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setSort(fromjson("{$natural: 1}")); ASSERT_OK(qr->validate()); @@ -272,7 +272,7 @@ TEST(ExpressionOptimizeTest, IsValidGeoNearNaturalSort) { // SERVER-14366 TEST(ExpressionOptimizeTest, IsValidGeoNearNaturalHint) { // Filter inside QueryRequest is not used. - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setHint(fromjson("{$natural: 1}")); ASSERT_OK(qr->validate()); diff --git a/src/mongo/db/modules/eloq/CMakeLists.txt b/src/mongo/db/modules/eloq/CMakeLists.txt index 65e9b5e076..4c2e92bfbf 100644 --- a/src/mongo/db/modules/eloq/CMakeLists.txt +++ b/src/mongo/db/modules/eloq/CMakeLists.txt @@ -47,3 +47,10 @@ endif() # --- Include build system modules --- add_subdirectory(data_substrate) + +# The pinned yaml-cpp 0.7.0 uses fixed-width integers in emitterutils.cpp without +# including . GCC 15 no longer supplies it through unrelated headers. +# Keep the compatibility include local to this downloaded dependency; this module +# is shared by the top-level CMake and historical SCons dependency builds. +target_compile_options(yaml-cpp PRIVATE + "$<$:SHELL:-include cstdint>") diff --git a/src/mongo/db/mongod_options.cpp b/src/mongo/db/mongod_options.cpp index 9eb6fe90ab..575a7a4e56 100644 --- a/src/mongo/db/mongod_options.cpp +++ b/src/mongo/db/mongod_options.cpp @@ -875,6 +875,18 @@ Status storeMongodOptions(const moe::Environment& params) { storageGlobalParams.engine = params["storage.engine"].as(); storageGlobalParams.engineSetByUser = true; } +#ifdef ELOQDOC_STANDALONE + if (params.count("replication.replSetName") || params.count("replication.replSet") || + params.count("sharding.clusterRole")) { + return Status(ErrorCodes::InvalidOptions, + "This EloqDoc build supports standalone mode only; --replSet, --shardsvr " + "and --configsvr are not supported"); + } + // The CMake server retains normal option parsing, but only the Eloq engine is linked. + if (!storageGlobalParams.engineSetByUser) { + storageGlobalParams.engine = "eloq"; + } +#endif if (params.count("storage.dbPath")) { storageGlobalParams.dbpath = params["storage.dbPath"].as(); diff --git a/src/mongo/db/nesting_depth_test.cpp b/src/mongo/db/nesting_depth_test.cpp index b6d325d299..1d1f2076ab 100644 --- a/src/mongo/db/nesting_depth_test.cpp +++ b/src/mongo/db/nesting_depth_test.cpp @@ -35,19 +35,55 @@ #include "mongo/bson/json.h" #include "mongo/client/connection_string.h" #include "mongo/executor/network_interface_integration_fixture.h" +#include "mongo/rpc/get_status_from_command_result.h" +#include "mongo/unittest/bson_test_util.h" #include "mongo/util/concurrency/thread_pool.h" namespace mongo { namespace executor { namespace { +constexpr auto kCollectionName = "depthTest"; + class NestingDepthFixture : public NetworkInterfaceIntegrationFixture { public: void setUp() final { startNet(); + RemoteCommandRequest request{fixture().getServers()[0], "admin", BSON("serverStatus" << 1), + BSONObj(), nullptr}; + auto response = runCommandSync(request); + ASSERT_OK(response.status); + ASSERT_OK(getStatusFromCommandResult(response.data)); + _usesEloq = response.data["storageEngine"].Obj()["name"].String() == "eloq"; + assertCommandOK(kCollectionName, BSON("dropDatabase" << 1)); } -}; -constexpr auto kCollectionName = "depthTest"; + void assertUpdateDepthError(const BSONObj& command) { + auto before = readDocument(); + if (_usesEloq) { + // Eloq's command-level transaction aborts the whole update command on error. + // Do not swallow that error into writeErrors and commit a partial transaction. + assertCommandFailsOnServer(kCollectionName, command, ErrorCodes::Overflow); + } else { + assertWriteError(kCollectionName, command, ErrorCodes::Overflow); + } + ASSERT_BSONOBJ_EQ(before, readDocument()); + } + +private: + BSONObj readDocument() { + RemoteCommandRequest request{fixture().getServers()[0], kCollectionName, + BSON("find" << kCollectionName << "singleBatch" << true), + BSONObj(), nullptr}; + auto response = runCommandSync(request); + ASSERT_OK(response.status); + ASSERT_OK(getStatusFromCommandResult(response.data)); + auto batch = response.data["cursor"].Obj()["firstBatch"].Array(); + ASSERT_EQ(1U, batch.size()); + return batch.front().Obj().getOwned(); + } + + bool _usesEloq = false; +}; /** * Appends an object to 'builder' that is nested 'depth' levels deep. @@ -274,7 +310,7 @@ TEST_F(NestingDepthFixture, CannotUpdateDocumentToExceedDepthLimit) { BSONObjBuilder updateCmd; appendUpdateCommandWithNestedDocuments( &updateCmd, largeButValidDepth, BSONDepth::getMaxDepthForUserStorage() + 1); - assertWriteError(kCollectionName, updateCmd.obj(), ErrorCodes::Overflow); + assertUpdateDepthError(updateCmd.obj()); } TEST_F(NestingDepthFixture, CanReplaceDocumentIfItStaysWithinDepthLimit) { @@ -308,7 +344,7 @@ TEST_F(NestingDepthFixture, CannotReplaceDocumentToExceedDepthLimit) { BSONObjBuilder updateCmd; appendUpdateReplaceCommandWithNestedDocuments( &updateCmd, largeButValidDepth, BSONDepth::getMaxDepthForUserStorage() + 1); - assertWriteError(kCollectionName, updateCmd.obj(), ErrorCodes::Overflow); + assertUpdateDepthError(updateCmd.obj()); } /** @@ -435,7 +471,7 @@ TEST_F(NestingDepthFixture, CannotUpdateArrayToExceedDepthLimit) { BSONObjBuilder updateCmd; appendUpdateCommandWithNestedArrays( &updateCmd, largeButValidDepth, BSONDepth::getMaxDepthForUserStorage() + 1); - assertWriteError(kCollectionName, updateCmd.obj(), ErrorCodes::Overflow); + assertUpdateDepthError(updateCmd.obj()); } TEST_F(NestingDepthFixture, CanReplaceArrayIfItStaysWithinDepthLimit) { @@ -469,7 +505,7 @@ TEST_F(NestingDepthFixture, CannotReplaceArrayToExceedDepthLimit) { BSONObjBuilder updateCmd; appendUpdateReplaceCommandWithNestedArrays( &updateCmd, largeButValidDepth, BSONDepth::getMaxDepthForUserStorage() + 1); - assertWriteError(kCollectionName, updateCmd.obj(), ErrorCodes::Overflow); + assertUpdateDepthError(updateCmd.obj()); } } // namespace } // namespace executor diff --git a/src/mongo/db/op_observer_noop.h b/src/mongo/db/op_observer_noop.h index 9e1a496660..f12830917a 100644 --- a/src/mongo/db/op_observer_noop.h +++ b/src/mongo/db/op_observer_noop.h @@ -61,7 +61,7 @@ class OpObserverNoop : public OpObserver { const BSONObj& msgObj, const boost::optional o2MsgObj) override {} void onCreateCollection(OperationContext* opCtx, - Collection* coll, + Collection::Uptr coll, const NamespaceString& collectionName, const CollectionOptions& options, const BSONObj& idIndex, diff --git a/src/mongo/db/operation_context.cpp b/src/mongo/db/operation_context.cpp index f7fc4ce5f6..0941629cd1 100644 --- a/src/mongo/db/operation_context.cpp +++ b/src/mongo/db/operation_context.cpp @@ -386,9 +386,7 @@ StatusWith OperationContext::waitForConditionOrInterruptNoAsser cv.wait(m); return stdx::cv_status::no_timeout; } - // return getServiceContext()->getPreciseClockSource()->waitForConditionUntil(cv, m, - // deadline); - return cv.wait_until(m, deadline); + return getServiceContext()->getPreciseClockSource()->waitForConditionUntil(cv, m, deadline); }(); // Continue waiting on cv until no other thread is attempting to kill this one. diff --git a/src/mongo/db/operation_context_test.cpp b/src/mongo/db/operation_context_test.cpp index b779102b70..f733a10b0d 100644 --- a/src/mongo/db/operation_context_test.cpp +++ b/src/mongo/db/operation_context_test.cpp @@ -29,12 +29,14 @@ #include "mongo/platform/basic.h" #include +#include #include "mongo/db/client.h" #include "mongo/db/json.h" #include "mongo/db/logical_session_id.h" #include "mongo/db/operation_context.h" #include "mongo/db/operation_context_group.h" +#include "mongo/db/operation_time_tracker.h" #include "mongo/db/service_context.h" #include "mongo/stdx/future.h" #include "mongo/stdx/memory.h" @@ -49,6 +51,27 @@ namespace mongo { namespace { +TEST(CoroutineConditionVariable, NativeThreadWaitForExpiresWithoutNotification) { + coro::Mutex mutex; + coro::ConditionVariable cv; + stdx::unique_lock lock(mutex); + const auto started = std::chrono::steady_clock::now(); + const auto delay = std::chrono::milliseconds(5); + ASSERT_FALSE(cv.wait_for(lock, delay, [] { return false; })); + ASSERT_TRUE(std::chrono::steady_clock::now() >= started + delay); + ASSERT_TRUE(lock.owns_lock()); +} + +TEST(CoroutineConditionVariable, NativeThreadDateDeadlineExpiresWithoutNotification) { + coro::Mutex mutex; + coro::ConditionVariable cv; + stdx::unique_lock lock(mutex); + auto deadline = Date_t::now() + Milliseconds(5); + while (cv.wait_until(lock, deadline) != stdx::cv_status::timeout) {} + ASSERT_TRUE(Date_t::now() >= deadline); + ASSERT_TRUE(lock.owns_lock()); +} + using unittest::assertGet; std::ostream& operator<<(std::ostream& os, stdx::cv_status cvStatus) { @@ -84,6 +107,79 @@ TEST(OperationContextTest, NoSessionIdNoTransactionNumber) { ASSERT(!opCtx->getTxnNumber()); } +TEST(OperationContextTest, NoStorageEngineLeavesRecoveryUnitUnset) { + auto serviceCtx = ServiceContext::make(); + auto client = serviceCtx->makeClient("OperationContextTest"); + ASSERT_FALSE(serviceCtx->getStorageEngine()); + // Exercise creation and repeated pool return/reuse without a test storage observer. + for (int i = 0; i < 64; ++i) { + auto opCtx = client->makeOperationContext(); + ASSERT_FALSE(opCtx->recoveryUnit()); + } +} + +TEST(OperationContextTest, PoolReleaseReusesUnsharedOperationTimeTracker) { + auto serviceCtx = ServiceContext::make(); + auto client = serviceCtx->makeClient("OperationContextTest"); + auto opCtx = client->makeOperationContext(); + auto tracker = OperationTimeTracker::get(opCtx.get()); + auto* original = tracker.get(); + // A weak reference observes lifetime without preventing exclusive-ownership reuse. + std::weak_ptr lifetime = tracker; + tracker->updateOperationTime(LogicalTime(Timestamp(15))); + tracker.reset(); + + for (int i = 0; i < 10; ++i) { + ASSERT_EQ(1, lifetime.use_count()); + opCtx->resetAllDecorations(); + // This also rules out destroy/reallocate-at-the-same-address implementations. + ASSERT_FALSE(lifetime.expired()); + auto reused = OperationTimeTracker::get(opCtx.get()); + ASSERT_EQ(original, reused.get()); + ASSERT_EQ(LogicalTime::kUninitialized, reused->getMaxOperationTime()); + reused->updateOperationTime(LogicalTime(Timestamp(5))); + ASSERT_EQ(LogicalTime(Timestamp(5)), reused->getMaxOperationTime()); + } + + // Exercise the actual ServiceContext deleter -> pool -> decoration-reset path too. + opCtx.reset(); + ASSERT_FALSE(lifetime.expired()); + ASSERT_EQ(LogicalTime::kUninitialized, lifetime.lock()->getMaxOperationTime()); +} + +TEST(OperationContextTest, PoolReleasePreservesSharedOperationTimeTracker) { + auto serviceCtx = ServiceContext::make(); + auto client = serviceCtx->makeClient("OperationContextTest"); + auto opCtx = client->makeOperationContext(); + auto previous = OperationTimeTracker::get(opCtx.get()); + const LogicalTime previousTime(Timestamp(15)); + previous->updateOperationTime(previousTime); + + opCtx->resetAllDecorations(); + auto current = OperationTimeTracker::get(opCtx.get()); + ASSERT_NE(previous, current); + ASSERT_EQ(previousTime, previous->getMaxOperationTime()); + ASSERT_EQ(LogicalTime::kUninitialized, current->getMaxOperationTime()); + previous->updateOperationTime(LogicalTime(Timestamp(30))); + ASSERT_EQ(LogicalTime::kUninitialized, current->getMaxOperationTime()); + current->updateOperationTime(LogicalTime(Timestamp(5))); + ASSERT_EQ(LogicalTime(Timestamp(30)), previous->getMaxOperationTime()); + + // Once the replacement is exclusively owned, subsequent recycling must reuse it. + std::weak_ptr lifetime = current; + current.reset(); + opCtx->resetAllDecorations(); + ASSERT_FALSE(lifetime.expired()); + ASSERT_EQ(LogicalTime::kUninitialized, lifetime.lock()->getMaxOperationTime()); + + // A retained tracker also survives the real pool-return path with its state intact. + current = OperationTimeTracker::get(opCtx.get()); + current->updateOperationTime(LogicalTime(Timestamp(7))); + opCtx.reset(); + ASSERT_EQ(LogicalTime(Timestamp(7)), current->getMaxOperationTime()); + ASSERT_EQ(LogicalTime(Timestamp(30)), previous->getMaxOperationTime()); +} + TEST(OperationContextTest, SessionIdNoTransactionNumber) { auto serviceCtx = ServiceContext::make(); auto client = serviceCtx->makeClient("OperationContextTest"); diff --git a/src/mongo/db/operation_time_tracker.cpp b/src/mongo/db/operation_time_tracker.cpp index 7bf6f97f24..2bec454783 100644 --- a/src/mongo/db/operation_time_tracker.cpp +++ b/src/mongo/db/operation_time_tracker.cpp @@ -36,7 +36,14 @@ namespace { struct OperationTimeTrackerHolder { OperationTimeTrackerHolder() : opTimeTracker(std::make_shared()) {} void reset() { - opTimeTracker.reset(); + // OperationContext recycling has exclusive access to this holder. Reuse the tracker + // unless another shared owner (e.g. a remote callback) still needs the previous state. + invariant(opTimeTracker); + if (opTimeTracker.use_count() == 1) { + opTimeTracker->reset(); + } else { + opTimeTracker = std::make_shared(); + } } static const OperationContext::Decoration get; std::shared_ptr opTimeTracker; @@ -47,7 +54,7 @@ const OperationContext::Decoration OperationTimeTrac } // namespace std::shared_ptr OperationTimeTracker::get(OperationContext* opCtx) { - auto timeTrackerHolder = OperationTimeTrackerHolder::get(opCtx); + auto& timeTrackerHolder = OperationTimeTrackerHolder::get(opCtx); invariant(timeTrackerHolder.opTimeTracker); return timeTrackerHolder.opTimeTracker; } @@ -64,4 +71,9 @@ void OperationTimeTracker::updateOperationTime(LogicalTime newTime) { } } +void OperationTimeTracker::reset() { + stdx::lock_guard lock(_mutex); + _maxOperationTime = LogicalTime::kUninitialized; +} + } // namespace mongo diff --git a/src/mongo/db/operation_time_tracker.h b/src/mongo/db/operation_time_tracker.h index d1479776ad..b8c62a11f0 100644 --- a/src/mongo/db/operation_time_tracker.h +++ b/src/mongo/db/operation_time_tracker.h @@ -54,6 +54,9 @@ class OperationTimeTracker { */ void updateOperationTime(LogicalTime newTime); + // Clear state for reuse only when no other operation/callback owns this tracker. + void reset(); + private: // protects _maxOperationTime mutable stdx::mutex _mutex; diff --git a/src/mongo/db/operation_time_tracker_test.cpp b/src/mongo/db/operation_time_tracker_test.cpp index f582c891c6..39d823a1ef 100644 --- a/src/mongo/db/operation_time_tracker_test.cpp +++ b/src/mongo/db/operation_time_tracker_test.cpp @@ -30,11 +30,31 @@ #include "mongo/db/logical_time.h" #include "mongo/db/operation_time_tracker.h" +#include "mongo/db/service_context_test_fixture.h" #include "mongo/unittest/unittest.h" namespace mongo { namespace { +using OperationTimeTrackerTest = ServiceContextTest; + +TEST_F(OperationTimeTrackerTest, ResetDecorationsCreatesIndependentTracker) { + auto opCtx = makeOperationContext(); + auto previous = OperationTimeTracker::get(opCtx.get()); + const LogicalTime previousTime(Timestamp(15)); + previous->updateOperationTime(previousTime); + ASSERT_EQ(previous, OperationTimeTracker::get(opCtx.get())); + + // OperationContext's pool deleter resets decorations without destroying the context. + opCtx->resetAllDecorations(); + auto current = OperationTimeTracker::get(opCtx.get()); + ASSERT_NE(previous, current); + ASSERT_EQ(LogicalTime::kUninitialized, current->getMaxOperationTime()); + current->updateOperationTime(LogicalTime(Timestamp(5))); + ASSERT_EQ(previousTime, previous->getMaxOperationTime()); + ASSERT_EQ(current, OperationTimeTracker::get(opCtx.get())); +} + TEST(OperationTimeTracker, UnintializedMaxOperationTime) { OperationTimeTracker opTimeTracker; ASSERT_TRUE(opTimeTracker.getMaxOperationTime() == LogicalTime::kUninitialized); diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index fbdc846e83..6e08719ccf 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -307,6 +307,7 @@ pipelineeEnv.Library( 'document_source_unwind.cpp', 'mongo_process_common.cpp', 'pipeline.cpp', + 'pipeline_sharded.cpp', 'sequential_document_cache.cpp', 'tee_buffer.cpp', ], diff --git a/src/mongo/db/pipeline/aggregation_context_fixture.h b/src/mongo/db/pipeline/aggregation_context_fixture.h index 4f53f2fc55..003544f25f 100644 --- a/src/mongo/db/pipeline/aggregation_context_fixture.h +++ b/src/mongo/db/pipeline/aggregation_context_fixture.h @@ -51,7 +51,7 @@ class AggregationContextFixture : public ServiceContextTest { // Must instantiate ExpressionContext _after_ setting the TZ database on the service // context. // _expCtx = new ExpressionContext(_opCtx.get(), nullptr); - _expCtx = ObjectPool::newObjectRawPointer(_opCtx, nullptr); + _expCtx = ObjectPool::newObjectRawPointer(_opCtx.get(), nullptr); _expCtx->ns = std::move(nss); } diff --git a/src/mongo/db/pipeline/pipeline.cpp b/src/mongo/db/pipeline/pipeline.cpp index 12598b1c57..b8bf88cf70 100644 --- a/src/mongo/db/pipeline/pipeline.cpp +++ b/src/mongo/db/pipeline/pipeline.cpp @@ -38,12 +38,10 @@ #include "mongo/db/jsobj.h" #include "mongo/db/operation_context.h" #include "mongo/db/pipeline/accumulator.h" -#include "mongo/db/pipeline/cluster_aggregation_planner.h" #include "mongo/db/pipeline/document.h" #include "mongo/db/pipeline/document_source.h" #include "mongo/db/pipeline/document_source_geo_near.h" #include "mongo/db/pipeline/document_source_match.h" -#include "mongo/db/pipeline/document_source_merge_cursors.h" #include "mongo/db/pipeline/document_source_out.h" #include "mongo/db/pipeline/document_source_project.h" #include "mongo/db/pipeline/document_source_sort.h" @@ -314,25 +312,6 @@ void Pipeline::dispose(OperationContext* opCtx) { } } -std::unique_ptr Pipeline::splitForSharded() { - invariant(!isSplitForShards()); - invariant(!isSplitForMerge()); - - // Create and initialize the shard spec we'll return. We start with an empty pipeline on the - // shards and all work being done in the merger. Optimizations can move operations between - // the pipelines to be more efficient. - std::unique_ptr shardPipeline(new Pipeline(pCtx), - PipelineDeleter(pCtx->opCtx)); - - cluster_aggregation_planner::performSplitPipelineOptimizations(shardPipeline.get(), this); - shardPipeline->_splitState = SplitState::kSplitForShards; - _splitState = SplitState::kSplitForMerge; - - stitch(); - - return shardPipeline; -} - BSONObj Pipeline::getInitialQuery() const { if (_sources.empty()) return BSONObj(); diff --git a/src/mongo/db/pipeline/pipeline_sharded.cpp b/src/mongo/db/pipeline/pipeline_sharded.cpp new file mode 100644 index 0000000000..c673b8d121 --- /dev/null +++ b/src/mongo/db/pipeline/pipeline_sharded.cpp @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2011 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/pipeline/pipeline.h" + +#include "mongo/db/pipeline/cluster_aggregation_planner.h" +#include "mongo/db/pipeline/expression_context.h" +#include "mongo/util/assert_util.h" + +namespace mongo { + +// Only builds with MongoDB sharding link this translation unit. The common Pipeline +// implementation can be linked without referencing the distributed aggregation planner. +std::unique_ptr Pipeline::splitForSharded() { + invariant(!isSplitForShards()); + invariant(!isSplitForMerge()); + + // Create and initialize the shard spec we'll return. We start with an empty pipeline on the + // shards and all work being done in the merger. Optimizations can move operations between + // the pipelines to be more efficient. + std::unique_ptr shardPipeline(new Pipeline(pCtx), + PipelineDeleter(pCtx->opCtx)); + + cluster_aggregation_planner::performSplitPipelineOptimizations(shardPipeline.get(), this); + shardPipeline->_splitState = SplitState::kSplitForShards; + _splitState = SplitState::kSplitForMerge; + + stitch(); + + return shardPipeline; +} + +} // namespace mongo diff --git a/src/mongo/db/query/canonical_query_test.cpp b/src/mongo/db/query/canonical_query_test.cpp index 8f7a35ad8a..3c41fb95a1 100644 --- a/src/mongo/db/query/canonical_query_test.cpp +++ b/src/mongo/db/query/canonical_query_test.cpp @@ -175,13 +175,13 @@ TEST(CanonicalQueryTest, SortTreeNumChildrenComparison) { /** * Utility function to create a CanonicalQuery */ -unique_ptr canonicalize(const char* queryStr, +CanonicalQuery::UPtr canonicalize(const char* queryStr, MatchExpressionParser::AllowedFeatureSet allowedFeatures = MatchExpressionParser::kDefaultSpecialFeatures) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); auto statusWithCQ = CanonicalQuery::canonicalize( @@ -191,13 +191,13 @@ unique_ptr canonicalize(const char* queryStr, return std::move(statusWithCQ.getValue()); } -std::unique_ptr canonicalize(const char* queryStr, +CanonicalQuery::UPtr canonicalize(const char* queryStr, const char* sortStr, const char* projStr) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projStr)); @@ -210,7 +210,7 @@ std::unique_ptr canonicalize(const char* queryStr, * Test function for CanonicalQuery::normalize. */ void testNormalizeQuery(const char* queryStr, const char* expectedExprStr) { - unique_ptr cq(canonicalize(queryStr)); + CanonicalQuery::UPtr cq(canonicalize(queryStr)); MatchExpression* me = cq->root(); BSONObj expectedExprObj = fromjson(expectedExprStr); unique_ptr expectedExpr(parseMatchExpression(expectedExprObj)); @@ -289,7 +289,7 @@ TEST(CanonicalQueryTest, CanonicalQueryFromQRWithNoCollation) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); auto cq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), std::move(qr))); ASSERT_TRUE(cq->getCollator() == nullptr); } @@ -298,7 +298,7 @@ TEST(CanonicalQueryTest, CanonicalQueryFromQRWithCollation) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setCollation(BSON("locale" << "reverse")); auto cq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), std::move(qr))); @@ -310,7 +310,7 @@ TEST(CanonicalQueryTest, CanonicalQueryFromBaseQueryWithNoCollation) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{$or:[{a:1,b:1},{a:1,c:1}]}")); auto baseCq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), std::move(qr))); MatchExpression* firstClauseExpr = baseCq->root()->getChild(0); @@ -323,7 +323,7 @@ TEST(CanonicalQueryTest, CanonicalQueryFromBaseQueryWithCollation) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{$or:[{a:1,b:1},{a:1,c:1}]}")); qr->setCollation(BSON("locale" << "reverse")); @@ -339,7 +339,7 @@ TEST(CanonicalQueryTest, SettingCollatorUpdatesCollatorAndMatchExpression) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson("{a: 'foo', b: {$in: ['bar', 'baz']}}")); auto cq = assertGet(CanonicalQuery::canonicalize(opCtx.get(), std::move(qr))); ASSERT_EQUALS(2U, cq->root()->numChildren()); @@ -368,7 +368,7 @@ TEST(CanonicalQueryTest, SettingCollatorUpdatesCollatorAndMatchExpression) { } TEST(CanonicalQueryTest, NorWithOneChildNormalizedToNot) { - unique_ptr cq(canonicalize("{$nor: [{a: 1}]}")); + CanonicalQuery::UPtr cq(canonicalize("{$nor: [{a: 1}]}")); auto root = cq->root(); ASSERT_EQ(MatchExpression::NOT, root->matchType()); ASSERT_EQ(1U, root->numChildren()); @@ -376,13 +376,13 @@ TEST(CanonicalQueryTest, NorWithOneChildNormalizedToNot) { } TEST(CanonicalQueryTest, NorWithTwoChildrenNotNormalized) { - unique_ptr cq(canonicalize("{$nor: [{a: 1}, {b: 1}]}")); + CanonicalQuery::UPtr cq(canonicalize("{$nor: [{a: 1}, {b: 1}]}")); auto root = cq->root(); ASSERT_EQ(MatchExpression::NOR, root->matchType()); } TEST(CanonicalQueryTest, NorWithOneChildNormalizedAfterNormalizingChild) { - unique_ptr cq(canonicalize("{$nor: [{$or: [{a: 1}]}]}")); + CanonicalQuery::UPtr cq(canonicalize("{$nor: [{$or: [{a: 1}]}]}")); auto root = cq->root(); ASSERT_EQ(MatchExpression::NOT, root->matchType()); ASSERT_EQ(1U, root->numChildren()); diff --git a/src/mongo/db/query/collation/SConscript b/src/mongo/db/query/collation/SConscript index 09f8043a4c..82f48f9d92 100644 --- a/src/mongo/db/query/collation/SConscript +++ b/src/mongo/db/query/collation/SConscript @@ -100,19 +100,7 @@ env.CppUnitTest( ], ) -env.CppUnitTest( - target="collator_factory_icu_test", - source=[ - "collator_factory_icu_locales_test.cpp", - "collator_factory_icu_test.cpp", - ], - LIBDEPS=[ - "collator_icu", - ], -) - -# The collator_icu library and the collator_interface_icu_test unit tests need an environment which -# has access to the third-party ICU headers. +# The collator library and both ICU test targets must use matching ICU headers and defines. icuEnv = env.Clone() if not use_system_version_of_library("icu"): @@ -148,6 +136,17 @@ icuEnv.Library( ], ) +icuEnv.CppUnitTest( + target="collator_factory_icu_test", + source=[ + "collator_factory_icu_locales_test.cpp", + "collator_factory_icu_test.cpp", + ], + LIBDEPS=[ + "collator_icu", + ], +) + icuEnv.CppUnitTest( target="collator_interface_icu_test", source=[ diff --git a/src/mongo/db/query/collation/collator_factory_icu_test.cpp b/src/mongo/db/query/collation/collator_factory_icu_test.cpp index e2a393a787..742c2e9957 100644 --- a/src/mongo/db/query/collation/collator_factory_icu_test.cpp +++ b/src/mongo/db/query/collation/collator_factory_icu_test.cpp @@ -30,6 +30,8 @@ #include "mongo/db/query/collation/collator_factory_icu.h" +#include +#include #include "mongo/base/init.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/stdx/memory.h" @@ -827,9 +829,9 @@ TEST(CollatorFactoryICUTest, VersionFieldParsesSuccessfully) { auto collator = factory.makeFromBSON(BSON("locale" << "en_US" << "version" - << "57.1")); + << U_ICU_VERSION)); ASSERT_OK(collator.getStatus()); - ASSERT_EQ("57.1", collator.getValue()->getSpec().version); + ASSERT_EQ(U_ICU_VERSION, collator.getValue()->getSpec().version); } TEST(CollatorFactoryICUTest, VersionFieldPopulatedWhenOmitted) { @@ -837,7 +839,7 @@ TEST(CollatorFactoryICUTest, VersionFieldPopulatedWhenOmitted) { auto collator = factory.makeFromBSON(BSON("locale" << "en_US")); ASSERT_OK(collator.getStatus()); - ASSERT_EQ("57.1", collator.getValue()->getSpec().version); + ASSERT_EQ(U_ICU_VERSION, collator.getValue()->getSpec().version); } TEST(CollatorFactoryICUTest, NonStringVersionFieldFailsToParse) { @@ -1212,12 +1214,24 @@ TEST(CollatorFactoryICUTest, AliasNoForNorwegianBokmalLocaleNotSupported) { .getStatus()); } -TEST(CollatorFactoryICUTest, TraditionalSpanishAliasNotSupported) { - CollatorFactoryICU factory; - ASSERT_NOT_OK(factory - .makeFromBSON(BSON("locale" - << "es__TRADITIONAL")) - .getStatus()); +TEST(CollatorFactoryICUTest, TraditionalSpanishAliasMustMatchICUValidLocale) { + // ICU versions differ in whether this is recognized verbatim or canonicalized to another + // locale. The factory must accept it only when ICU's valid locale matches the request. + const char* localeID = "es__TRADITIONAL"; + const auto locale = icu::Locale::createFromName(localeID); + UErrorCode status = U_ZERO_ERROR; + std::unique_ptr reference(icu::Collator::createInstance(locale, status)); + ASSERT_TRUE(U_SUCCESS(status)); + const auto validLocale = reference->getLocale(ULOC_VALID_LOCALE, status); + ASSERT_TRUE(U_SUCCESS(status)); + CollatorFactoryICU factory; + auto result = factory.makeFromBSON(BSON("locale" << localeID)); + if (StringData(validLocale.getName()) == localeID) { + ASSERT_OK(result.getStatus()); + ASSERT_EQ(result.getValue()->getSpec().localeID, locale.getName()); + } else { + ASSERT_EQ(result.getStatus().code(), ErrorCodes::BadValue); + } } } // namespace diff --git a/src/mongo/db/query/collation/collator_interface_icu.h b/src/mongo/db/query/collation/collator_interface_icu.h index 9bb8382489..232f1d86a6 100644 --- a/src/mongo/db/query/collation/collator_interface_icu.h +++ b/src/mongo/db/query/collation/collator_interface_icu.h @@ -31,10 +31,11 @@ #include "mongo/db/query/collation/collator_interface.h" #include +#include -namespace icu { +U_NAMESPACE_BEGIN class Collator; -} // namespace icu +U_NAMESPACE_END namespace mongo { diff --git a/src/mongo/db/query/collation/collator_interface_icu_test.cpp b/src/mongo/db/query/collation/collator_interface_icu_test.cpp index 821321d793..9cafce8d7c 100644 --- a/src/mongo/db/query/collation/collator_interface_icu_test.cpp +++ b/src/mongo/db/query/collation/collator_interface_icu_test.cpp @@ -33,6 +33,8 @@ #include #include #include +#include +#include #include "mongo/unittest/unittest.h" @@ -92,6 +94,30 @@ void assertEqualEnUS(StringData left, StringData right) { assertEnUSComparison(right, left, ExpectedComparison::EQUAL); } +void assertMalformedUTF8MatchesICU(StringData input) { + // ICU releases differ in how many replacement characters represent an invalid + // sequence. Check the wrapper against this version's UTF-8 decoder, not ICU 57 data. + const auto decoded = icu::UnicodeString::fromUTF8( + icu::StringPiece(input.rawData(), input.size())); + std::string replaced; + decoded.toUTF8String(replaced); + ASSERT_FALSE(replaced.empty()); + ASSERT_NOT_EQUALS(input, StringData(replaced)); + assertEqualEnUS(input, replaced); +} + +std::string referenceSortKey(const icu::Collator& collator, StringData input) { + const auto decoded = icu::UnicodeString::fromUTF8( + icu::StringPiece(input.rawData(), input.size())); + const auto size = collator.getSortKey(decoded, nullptr, 0); + std::vector bytes(size); + ASSERT_EQUALS(size, collator.getSortKey(decoded, bytes.data(), size)); + ASSERT_GREATER_THAN(size, 0); + ASSERT_EQUALS(0, bytes.back()); + // Mongo's ComparisonKey excludes ICU's terminating zero byte. + return std::string(reinterpret_cast(bytes.data()), size - 1); +} + // Returns true if an ICU collator compares 'left' and 'right' as non-equal. Verifies that // the comparison is correct using the compare() function directly and using comparison keys. void assertNotEqualEnUS(StringData left, StringData right) { @@ -438,26 +464,26 @@ TEST(CollatorInterfaceICUTest, InvalidOneByteSeqAndTwoByteSeqCompareEqual) { TEST(CollatorInterfaceICUTest, OverlongASCIICharacterComparesEqualToReplacementCharacter) { // U+002F is the ASCII character "/", which should usually be represented as \x2F. The // representation \xC0\xAF is an unnecessary two-byte encoding of this codepoint. - assertEqualEnUS("\xC0\xAF", u8"\uFFFD"); + assertMalformedUTF8MatchesICU("\xC0\xAF"); } TEST(CollatorInterfaceICUTest, OverlongNullComparesEqualToReplacementCharacter) { // The two-byte sequence \xC0\x80 decodes to U+0000, which should instead be encoded using a // single null byte. - assertEqualEnUS("\xC0\x80", u8"\uFFFD"); + assertMalformedUTF8MatchesICU("\xC0\x80"); } TEST(CollatorInterfaceICUTest, IllegalCodePositionsCompareEqualToReplacementCharacter) { // U+D800 - assertEqualEnUS("\xED\xA0\x80", u8"\uFFFD"); + assertMalformedUTF8MatchesICU("\xED\xA0\x80"); // U+DBFF - assertEqualEnUS("\xED\xAF\xBF", u8"\uFFFD"); + assertMalformedUTF8MatchesICU("\xED\xAF\xBF"); // U+DFFF - assertEqualEnUS("\xED\xBF\xBF", u8"\uFFFD"); + assertMalformedUTF8MatchesICU("\xED\xBF\xBF"); // U+D800, U+DC00 - assertEqualEnUS("\xED\xA0\x80\xED\xB0\x80", u8"\uFFFD\uFFFD"); + assertMalformedUTF8MatchesICU("\xED\xA0\x80\xED\xB0\x80"); // U+DB80, U+DFFF - assertEqualEnUS("\xED\xAE\x80\xED\xBF\xBF", u8"\uFFFD\uFFFD"); + assertMalformedUTF8MatchesICU("\xED\xAE\x80\xED\xBF\xBF"); } TEST(CollatorInterfaceICUTest, UnexpectedTrailingContinuationByteComparesAsReplacementCharacter) { @@ -520,7 +546,7 @@ TEST(CollatorInterfaceICUTest, LastPossibleSequenceOfLengthNotEqualToReplacement TEST(CollatorInterfaceICUTest, CodePointBeyondLargestValidComparesEqualToReplacementCharacter) { // Largest valid code point is U+0010FFFF; U+001FFFFF is higher, and is the last possible valid // four byte sequence. - assertEqualEnUS("\xF7\xBF\xBF\xBF", u8"\uFFFD"); + assertMalformedUTF8MatchesICU("\xF7\xBF\xBF\xBF"); } TEST(CollatorInterfaceICUTest, StringsWithDifferentEmbeddedInvalidSequencesCompareEqual) { @@ -572,11 +598,13 @@ TEST(CollatorInterfaceICUTest, ComparisonKeysForEnUsCollatorCorrect) { std::unique_ptr coll( icu::Collator::createInstance(icu::Locale("en", "US"), status)); ASSERT(U_SUCCESS(status)); + const auto asciiKey = referenceSortKey(*coll, "abc"); + const auto accentedKey = referenceSortKey(*coll, "c\xC3\xB4t\xC3\xA9"); CollatorInterfaceICU icuCollator(collationSpec, std::move(coll)); - ASSERT_EQ(icuCollator.getComparisonKey("abc").getKeyData(), "\x29\x2B\x2D\x01\x07\x01\x07"); + ASSERT_EQ(icuCollator.getComparisonKey("abc").getKeyData(), asciiKey); ASSERT_EQ(icuCollator.getComparisonKey("c\xC3\xB4t\xC3\xA9").getKeyData(), - "\x2D\x45\x4F\x31\x01\x44\x8E\x44\x88\x01\x0A"); + accentedKey); } TEST(CollatorInterfaceICUTest, ComparisonKeysForFrCaCollatorCorrect) { @@ -586,11 +614,13 @@ TEST(CollatorInterfaceICUTest, ComparisonKeysForFrCaCollatorCorrect) { std::unique_ptr coll( icu::Collator::createInstance(icu::Locale("fr", "CA"), status)); ASSERT(U_SUCCESS(status)); + const auto asciiKey = referenceSortKey(*coll, "abc"); + const auto accentedKey = referenceSortKey(*coll, "c\xC3\xB4t\xC3\xA9"); CollatorInterfaceICU icuCollator(collationSpec, std::move(coll)); - ASSERT_EQ(icuCollator.getComparisonKey("abc").getKeyData(), "\x29\x2B\x2D\x01\x07\x01\x07"); + ASSERT_EQ(icuCollator.getComparisonKey("abc").getKeyData(), asciiKey); ASSERT_EQ(icuCollator.getComparisonKey("c\xC3\xB4t\xC3\xA9").getKeyData(), - "\x2D\x45\x4F\x31\x01\x88\x44\x8E\x06\x01\x0A"); + accentedKey); } } // namespace diff --git a/src/mongo/db/query/get_executor_test.cpp b/src/mongo/db/query/get_executor_test.cpp index 002d4a427b..83527ae5d4 100644 --- a/src/mongo/db/query/get_executor_test.cpp +++ b/src/mongo/db/query/get_executor_test.cpp @@ -47,20 +47,18 @@ using namespace mongo; namespace { -using std::unique_ptr; - static const NamespaceString nss("test.collection"); /** * Utility functions to create a CanonicalQuery */ -unique_ptr canonicalize(const char* queryStr, - const char* sortStr, - const char* projStr) { +CanonicalQuery::UPtr canonicalize(const char* queryStr, + const char* sortStr, + const char* projStr) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projStr)); @@ -94,7 +92,7 @@ void testAllowedIndices(std::vector indexes, QuerySettings querySettings; // getAllowedIndices should return false when query shape is not yet in query settings. - unique_ptr cq(canonicalize("{a: 1}", "{}", "{}")); + CanonicalQuery::UPtr cq(canonicalize("{a: 1}", "{}", "{}")); PlanCacheKey key = planCache.computeKey(*cq); ASSERT_FALSE(querySettings.getAllowedIndicesFilter(key)); diff --git a/src/mongo/db/query/plan_cache_test.cpp b/src/mongo/db/query/plan_cache_test.cpp index 30ec089160..b7e37ccd73 100644 --- a/src/mongo/db/query/plan_cache_test.cpp +++ b/src/mongo/db/query/plan_cache_test.cpp @@ -67,11 +67,11 @@ static const NamespaceString nss("test.collection"); /** * Utility functions to create a CanonicalQuery */ -unique_ptr canonicalize(const BSONObj& queryObj) { +CanonicalQuery::UPtr canonicalize(const BSONObj& queryObj) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(queryObj); const boost::intrusive_ptr expCtx; auto statusWithCQ = @@ -84,19 +84,19 @@ unique_ptr canonicalize(const BSONObj& queryObj) { return std::move(statusWithCQ.getValue()); } -unique_ptr canonicalize(const char* queryStr) { +CanonicalQuery::UPtr canonicalize(const char* queryStr) { BSONObj queryObj = fromjson(queryStr); return canonicalize(queryObj); } -unique_ptr canonicalize(const char* queryStr, +CanonicalQuery::UPtr canonicalize(const char* queryStr, const char* sortStr, const char* projStr, const char* collationStr) { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projStr)); @@ -112,7 +112,7 @@ unique_ptr canonicalize(const char* queryStr, return std::move(statusWithCQ.getValue()); } -unique_ptr canonicalize(const char* queryStr, +CanonicalQuery::UPtr canonicalize(const char* queryStr, const char* sortStr, const char* projStr, long long skip, @@ -123,7 +123,7 @@ unique_ptr canonicalize(const char* queryStr, QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projStr)); @@ -147,7 +147,7 @@ unique_ptr canonicalize(const char* queryStr, return std::move(statusWithCQ.getValue()); } -unique_ptr canonicalize(const char* queryStr, +CanonicalQuery::UPtr canonicalize(const char* queryStr, const char* sortStr, const char* projStr, long long skip, @@ -159,7 +159,7 @@ unique_ptr canonicalize(const char* queryStr, QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(fromjson(queryStr)); qr->setSort(fromjson(sortStr)); qr->setProj(fromjson(projStr)); @@ -275,12 +275,12 @@ void assertShouldNotCacheQuery(const CanonicalQuery& query) { } void assertShouldNotCacheQuery(const BSONObj& query) { - unique_ptr cq(canonicalize(query)); + CanonicalQuery::UPtr cq(canonicalize(query)); assertShouldNotCacheQuery(*cq); } void assertShouldNotCacheQuery(const char* queryStr) { - unique_ptr cq(canonicalize(queryStr)); + CanonicalQuery::UPtr cq(canonicalize(queryStr)); assertShouldNotCacheQuery(*cq); } @@ -291,12 +291,12 @@ void assertShouldNotCacheQuery(const char* queryStr) { */ TEST(PlanCacheTest, ShouldCacheQueryBasic) { - unique_ptr cq(canonicalize("{a: 1}")); + CanonicalQuery::UPtr cq(canonicalize("{a: 1}")); assertShouldCacheQuery(*cq); } TEST(PlanCacheTest, ShouldCacheQuerySort) { - unique_ptr cq(canonicalize("{}", "{a: -1}", "{_id: 0, a: 1}", "{}")); + CanonicalQuery::UPtr cq(canonicalize("{}", "{a: -1}", "{_id: 0, a: 1}", "{}")); assertShouldCacheQuery(*cq); } @@ -310,7 +310,7 @@ TEST(PlanCacheTest, ShouldCacheQuerySort) { * This should normally be handled by the IDHack runner. */ TEST(PlanCacheTest, ShouldNotCacheQueryCollectionScan) { - unique_ptr cq(canonicalize("{}")); + CanonicalQuery::UPtr cq(canonicalize("{}")); assertShouldNotCacheQuery(*cq); } @@ -320,7 +320,7 @@ TEST(PlanCacheTest, ShouldNotCacheQueryCollectionScan) { * Therefore, not much point in caching. */ TEST(PlanCacheTest, ShouldNotCacheQueryWithHint) { - unique_ptr cq( + CanonicalQuery::UPtr cq( canonicalize("{a: 1}", "{}", "{}", 0, 0, "{a: 1, b: 1}", "{}", "{}")); assertShouldNotCacheQuery(*cq); } @@ -329,7 +329,7 @@ TEST(PlanCacheTest, ShouldNotCacheQueryWithHint) { * Min queries are a specialized case of hinted queries */ TEST(PlanCacheTest, ShouldNotCacheQueryWithMin) { - unique_ptr cq(canonicalize("{a: 1}", "{}", "{}", 0, 0, "{}", "{a: 100}", "{}")); + CanonicalQuery::UPtr cq(canonicalize("{a: 1}", "{}", "{}", 0, 0, "{}", "{a: 100}", "{}")); assertShouldNotCacheQuery(*cq); } @@ -337,7 +337,7 @@ TEST(PlanCacheTest, ShouldNotCacheQueryWithMin) { * Max queries are non-cacheable for the same reasons as min queries. */ TEST(PlanCacheTest, ShouldNotCacheQueryWithMax) { - unique_ptr cq(canonicalize("{a: 1}", "{}", "{}", 0, 0, "{}", "{}", "{a: 100}")); + CanonicalQuery::UPtr cq(canonicalize("{a: 1}", "{}", "{}", 0, 0, "{}", "{}", "{a: 100}")); assertShouldNotCacheQuery(*cq); } @@ -346,7 +346,7 @@ TEST(PlanCacheTest, ShouldNotCacheQueryWithMax) { * the planner is able to come up with a cacheable solution. */ TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinLegacyCoordinates) { - unique_ptr cq( + CanonicalQuery::UPtr cq( canonicalize("{a: {$geoWithin: " "{$box: [[-180, -90], [180, 90]]}}}")); assertShouldCacheQuery(*cq); @@ -356,7 +356,7 @@ TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinLegacyCoordinates) { * $geoWithin queries with GeoJSON coordinates are supported by the index bounds builder. */ TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinJSONCoordinates) { - unique_ptr cq( + CanonicalQuery::UPtr cq( canonicalize("{a: {$geoWithin: " "{$geometry: {type: 'Polygon', coordinates: " "[[[0, 0], [0, 90], [90, 0], [0, 0]]]}}}}")); @@ -367,7 +367,7 @@ TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinJSONCoordinates) { * $geoWithin queries with both legacy and GeoJSON coordinates are cacheable. */ TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinLegacyAndJSONCoordinates) { - unique_ptr cq( + CanonicalQuery::UPtr cq( canonicalize("{$or: [{a: {$geoWithin: {$geometry: {type: 'Polygon', " "coordinates: [[[0, 0], [0, 90], " "[90, 0], [0, 0]]]}}}}," @@ -379,7 +379,7 @@ TEST(PlanCacheTest, ShouldCacheQueryWithGeoWithinLegacyAndJSONCoordinates) { * $geoIntersects queries are always cacheable because they support GeoJSON coordinates only. */ TEST(PlanCacheTest, ShouldCacheQueryWithGeoIntersects) { - unique_ptr cq( + CanonicalQuery::UPtr cq( canonicalize("{a: {$geoIntersects: " "{$geometry: {type: 'Point', coordinates: " "[10.0, 10.0]}}}}")); @@ -391,7 +391,7 @@ TEST(PlanCacheTest, ShouldCacheQueryWithGeoIntersects) { * between flat and spherical queries. */ TEST(PlanCacheTest, ShouldNotCacheQueryWithGeoNear) { - unique_ptr cq( + CanonicalQuery::UPtr cq( canonicalize("{a: {$geoNear: {$geometry: {type: 'Point'," "coordinates: [0,0]}, $maxDistance:100}}}")); assertShouldCacheQuery(*cq); @@ -403,7 +403,7 @@ TEST(PlanCacheTest, ShouldNotCacheQueryWithGeoNear) { * non-winning plans. */ TEST(PlanCacheTest, ShouldNotCacheQueryExplain) { - unique_ptr cq(canonicalize("{a: 1}", + CanonicalQuery::UPtr cq(canonicalize("{a: 1}", "{}", "{}", 0, @@ -421,7 +421,7 @@ TEST(PlanCacheTest, ShouldNotCacheQueryExplain) { // Adding an empty vector of query solutions should fail. TEST(PlanCacheTest, AddEmptySolutions) { PlanCache planCache; - unique_ptr cq(canonicalize("{a: 1}")); + CanonicalQuery::UPtr cq(canonicalize("{a: 1}")); std::vector solns; unique_ptr decision(createDecision(1U)); QueryTestServiceContext serviceContext; @@ -430,7 +430,7 @@ TEST(PlanCacheTest, AddEmptySolutions) { TEST(PlanCacheTest, AddValidSolution) { PlanCache planCache; - unique_ptr cq(canonicalize("{a: 1}")); + CanonicalQuery::UPtr cq(canonicalize("{a: 1}")); QuerySolution qs; qs.cacheData.reset(new SolutionCacheData()); qs.cacheData->tree.reset(new PlanCacheIndexTree()); @@ -557,7 +557,7 @@ class CachePlanSelectionTest : public mongo::unittest::Test { // Clean up any previous state from a call to runQueryFull or runQueryAsCommand. solns.clear(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(query); qr->setSort(sort); qr->setProj(proj); @@ -591,7 +591,7 @@ class CachePlanSelectionTest : public mongo::unittest::Test { solns.clear(); const bool isExplain = false; - std::unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); const boost::intrusive_ptr expCtx; @@ -664,7 +664,7 @@ class CachePlanSelectionTest : public mongo::unittest::Test { QueryTestServiceContext serviceContext; auto opCtx = serviceContext.makeOperationContext(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(query); qr->setSort(sort); qr->setProj(proj); @@ -677,7 +677,7 @@ class CachePlanSelectionTest : public mongo::unittest::Test { ExtensionsCallbackNoop(), MatchExpressionParser::kAllowAllSpecialFeatures); ASSERT_OK(statusWithCQ.getStatus()); - unique_ptr scopedCq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr scopedCq = std::move(statusWithCQ.getValue()); // Create a CachedSolution the long way.. // QuerySolution -> PlanCacheEntry -> CachedSolution @@ -1332,7 +1332,7 @@ void testComputeKey(const char* queryStr, const char* expectedStr) { PlanCache planCache; const char* collationStr = "{}"; - unique_ptr cq(canonicalize(queryStr, sortStr, projStr, collationStr)); + CanonicalQuery::UPtr cq(canonicalize(queryStr, sortStr, projStr, collationStr)); PlanCacheKey key = planCache.computeKey(*cq); PlanCacheKey expectedKey(expectedStr); if (key == expectedKey) { @@ -1412,11 +1412,11 @@ TEST(PlanCacheTest, ComputeKeyGeoWithin) { PlanCache planCache; // Legacy coordinates. - unique_ptr cqLegacy( + CanonicalQuery::UPtr cqLegacy( canonicalize("{a: {$geoWithin: " "{$box: [[-180, -90], [180, 90]]}}}")); // GeoJSON coordinates. - unique_ptr cqNew( + CanonicalQuery::UPtr cqNew( canonicalize("{a: {$geoWithin: " "{$geometry: {type: 'Polygon', coordinates: " "[[[0, 0], [0, 90], [90, 0], [0, 0]]]}}}}")); @@ -1468,9 +1468,9 @@ TEST(PlanCacheTest, ComputeKeySparseIndex) { nullptr, // filterExpr BSONObj())}); - unique_ptr cqEqNumber(canonicalize("{a: 0}}")); - unique_ptr cqEqString(canonicalize("{a: 'x'}}")); - unique_ptr cqEqNull(canonicalize("{a: null}}")); + CanonicalQuery::UPtr cqEqNumber(canonicalize("{a: 0}}")); + CanonicalQuery::UPtr cqEqString(canonicalize("{a: 'x'}}")); + CanonicalQuery::UPtr cqEqNull(canonicalize("{a: null}}")); // 'cqEqNumber' and 'cqEqString' get the same key, since both are compatible with this // index. @@ -1496,9 +1496,9 @@ TEST(PlanCacheTest, ComputeKeyPartialIndex) { filterExpr.get(), BSONObj())}); - unique_ptr cqGtNegativeFive(canonicalize("{f: {$gt: -5}}")); - unique_ptr cqGtZero(canonicalize("{f: {$gt: 0}}")); - unique_ptr cqGtFive(canonicalize("{f: {$gt: 5}}")); + CanonicalQuery::UPtr cqGtNegativeFive(canonicalize("{f: {$gt: -5}}")); + CanonicalQuery::UPtr cqGtZero(canonicalize("{f: {$gt: 0}}")); + CanonicalQuery::UPtr cqGtFive(canonicalize("{f: {$gt: 5}}")); // 'cqGtZero' and 'cqGtFive' get the same key, since both are compatible with this index. ASSERT_EQ(planCache.computeKey(*cqGtZero), planCache.computeKey(*cqGtFive)); @@ -1522,11 +1522,11 @@ TEST(PlanCacheTest, ComputeKeyCollationIndex) { entry.collator = &collator; planCache.notifyOfIndexEntries({entry}); - unique_ptr containsString(canonicalize("{a: 'abc'}")); - unique_ptr containsObject(canonicalize("{a: {b: 'abc'}}")); - unique_ptr containsArray(canonicalize("{a: ['abc', 'xyz']}")); - unique_ptr noStrings(canonicalize("{a: 5}")); - unique_ptr containsStringHasCollation( + CanonicalQuery::UPtr containsString(canonicalize("{a: 'abc'}")); + CanonicalQuery::UPtr containsObject(canonicalize("{a: {b: 'abc'}}")); + CanonicalQuery::UPtr containsArray(canonicalize("{a: ['abc', 'xyz']}")); + CanonicalQuery::UPtr noStrings(canonicalize("{a: 5}")); + CanonicalQuery::UPtr containsStringHasCollation( canonicalize("{a: 'abc'}", "{}", "{}", "{locale: 'mock_reverse_string'}")); // 'containsString', 'containsObject', and 'containsArray' have the same key, since none are @@ -1541,11 +1541,11 @@ TEST(PlanCacheTest, ComputeKeyCollationIndex) { // index. ASSERT_EQ(planCache.computeKey(*noStrings), planCache.computeKey(*containsStringHasCollation)); - unique_ptr inContainsString(canonicalize("{a: {$in: [1, 'abc', 2]}}")); - unique_ptr inContainsObject(canonicalize("{a: {$in: [1, {b: 'abc'}, 2]}}")); - unique_ptr inContainsArray(canonicalize("{a: {$in: [1, ['abc', 'xyz'], 2]}}")); - unique_ptr inNoStrings(canonicalize("{a: {$in: [1, 2]}}")); - unique_ptr inContainsStringHasCollation( + CanonicalQuery::UPtr inContainsString(canonicalize("{a: {$in: [1, 'abc', 2]}}")); + CanonicalQuery::UPtr inContainsObject(canonicalize("{a: {$in: [1, {b: 'abc'}, 2]}}")); + CanonicalQuery::UPtr inContainsArray(canonicalize("{a: {$in: [1, ['abc', 'xyz'], 2]}}")); + CanonicalQuery::UPtr inNoStrings(canonicalize("{a: {$in: [1, 2]}}")); + CanonicalQuery::UPtr inContainsStringHasCollation( canonicalize("{a: {$in: [1, 'abc', 2]}}", "{}", "{}", "{locale: 'mock_reverse_string'}")); // 'inContainsString', 'inContainsObject', and 'inContainsArray' have the same key, since none diff --git a/src/mongo/db/query/query_planner_test.cpp b/src/mongo/db/query/query_planner_test.cpp index 0a6a2d3aed..8d941b5fb5 100644 --- a/src/mongo/db/query/query_planner_test.cpp +++ b/src/mongo/db/query/query_planner_test.cpp @@ -4313,11 +4313,11 @@ TEST_F(QueryPlannerTest, CacheDataFromTaggedTreeFailsOnBadInput) { // No relevant index matching the index tag. relevantIndices.push_back(IndexEntry(BSON("a" << 1))); - auto qr = stdx::make_unique(NamespaceString("test.collection")); + auto qr = ObjectPool::newObject(NamespaceString("test.collection")); qr->setFilter(BSON("a" << 3)); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - std::unique_ptr scopedCq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr scopedCq = std::move(statusWithCQ.getValue()); scopedCq->root()->setTag(new IndexTag(1)); ASSERT_NOT_OK( @@ -4327,11 +4327,11 @@ TEST_F(QueryPlannerTest, CacheDataFromTaggedTreeFailsOnBadInput) { TEST_F(QueryPlannerTest, TagAccordingToCacheFailsOnBadInput) { const NamespaceString nss("test.collection"); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(BSON("a" << 3)); auto statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(qr)); ASSERT_OK(statusWithCQ.getStatus()); - std::unique_ptr scopedCq = std::move(statusWithCQ.getValue()); + CanonicalQuery::UPtr scopedCq = std::move(statusWithCQ.getValue()); std::unique_ptr indexTree(new PlanCacheIndexTree()); indexTree->setIndexEntry(IndexEntry(BSON("a" << 1), "a_1")); @@ -4356,7 +4356,7 @@ TEST_F(QueryPlannerTest, TagAccordingToCacheFailsOnBadInput) { ASSERT_OK(s); // Regenerate canonical query in order to clear tags. - auto newQR = stdx::make_unique(nss); + auto newQR = ObjectPool::newObject(nss); newQR->setFilter(BSON("a" << 3)); statusWithCQ = CanonicalQuery::canonicalize(opCtx.get(), std::move(newQR)); ASSERT_OK(statusWithCQ.getStatus()); diff --git a/src/mongo/db/query/query_planner_test_fixture.cpp b/src/mongo/db/query/query_planner_test_fixture.cpp index 63ec2465f5..c457fe5dd1 100644 --- a/src/mongo/db/query/query_planner_test_fixture.cpp +++ b/src/mongo/db/query/query_planner_test_fixture.cpp @@ -225,7 +225,7 @@ void QueryPlannerTest::runQueryFull(const BSONObj& query, solns.clear(); cq.reset(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(query); qr->setSort(sort); qr->setProj(proj); @@ -307,7 +307,7 @@ void QueryPlannerTest::runInvalidQueryFull(const BSONObj& query, solns.clear(); cq.reset(); - auto qr = stdx::make_unique(nss); + auto qr = ObjectPool::newObject(nss); qr->setFilter(query); qr->setSort(sort); qr->setProj(proj); @@ -346,7 +346,7 @@ void QueryPlannerTest::runQueryAsCommand(const BSONObj& cmdObj) { invariant(nss.isValid()); const bool isExplain = false; - std::unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); const boost::intrusive_ptr expCtx; @@ -371,7 +371,7 @@ void QueryPlannerTest::runInvalidQueryAsCommand(const BSONObj& cmdObj) { invariant(nss.isValid()); const bool isExplain = false; - std::unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); const boost::intrusive_ptr expCtx; diff --git a/src/mongo/db/query/query_planner_test_fixture.h b/src/mongo/db/query/query_planner_test_fixture.h index 2bdd7d5b16..6a9e15ed52 100644 --- a/src/mongo/db/query/query_planner_test_fixture.h +++ b/src/mongo/db/query/query_planner_test_fixture.h @@ -206,7 +206,7 @@ class QueryPlannerTest : public mongo::unittest::Test { QueryTestServiceContext serviceContext; ServiceContext::UniqueOperationContext opCtx; BSONObj queryObj; - std::unique_ptr cq; + CanonicalQuery::UPtr cq; QueryPlannerParams params; std::vector> solns; }; diff --git a/src/mongo/db/query/query_request_test.cpp b/src/mongo/db/query/query_request_test.cpp index c9354dfe1f..ae905d8e49 100644 --- a/src/mongo/db/query/query_request_test.cpp +++ b/src/mongo/db/query/query_request_test.cpp @@ -40,6 +40,7 @@ #include "mongo/db/pipeline/aggregation_request.h" #include "mongo/db/query/query_request.h" #include "mongo/db/service_context_test_fixture.h" +#include "mongo/db/storage/recovery_unit_noop.h" #include "mongo/unittest/unittest.h" namespace mongo { @@ -369,7 +370,7 @@ TEST(QueryRequestTest, ParseFromCommandWithOptions) { "maxScan: 1000}}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); // Make sure the values from the command BSON are reflected in the QR. @@ -384,7 +385,7 @@ TEST(QueryRequestTest, ParseFromCommandHintAsString) { "hint: 'foo_1'}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); BSONObj hintObj = qr->getHint(); @@ -423,7 +424,7 @@ TEST(QueryRequestTest, ParseFromCommandAllFlagsTrue) { "allowPartialResults: true}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); // Test that all the flags got set to true. @@ -443,7 +444,7 @@ TEST(QueryRequestTest, ParseFromCommandCommentWithValidMinMax) { "max: {a: 2}}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_EQUALS("the comment", qr->getComment()); @@ -469,7 +470,7 @@ TEST(QueryRequestTest, ParseFromCommandAllNonOptionFields) { "singleBatch: false}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); // Check the values inside the QR. @@ -501,7 +502,7 @@ TEST(QueryRequestTest, ParseFromCommandLargeLimit) { "limit: 8000000000}"); // 8 * 1000 * 1000 * 1000 const NamespaceString nss("test.testns"); const bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_EQUALS(8LL * 1000 * 1000 * 1000, *qr->getLimit()); @@ -514,7 +515,7 @@ TEST(QueryRequestTest, ParseFromCommandLargeBatchSize) { "batchSize: 8000000000}"); // 8 * 1000 * 1000 * 1000 const NamespaceString nss("test.testns"); const bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_EQUALS(8LL * 1000 * 1000 * 1000, *qr->getBatchSize()); @@ -527,7 +528,7 @@ TEST(QueryRequestTest, ParseFromCommandLargeSkip) { "skip: 8000000000}"); // 8 * 1000 * 1000 * 1000 const NamespaceString nss("test.testns"); const bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_EQUALS(8LL * 1000 * 1000 * 1000, *qr->getSkip()); @@ -817,7 +818,7 @@ TEST(QueryRequestTest, ParseFromCommandSkipIsZero) { "filter: {a: 3}}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_BSONOBJ_EQ(BSON("a" << 3), qr->getFilter()); ASSERT_FALSE(qr->getSkip()); @@ -841,7 +842,7 @@ TEST(QueryRequestTest, ParseFromCommandLimitIsZero) { "filter: {a: 3}}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_BSONOBJ_EQ(BSON("a" << 3), qr->getFilter()); ASSERT_FALSE(qr->getLimit()); @@ -862,7 +863,7 @@ TEST(QueryRequestTest, ParseFromCommandBatchSizeZero) { BSONObj cmdObj = fromjson("{find: 'testns', batchSize: 0}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT(qr->getBatchSize()); @@ -875,7 +876,7 @@ TEST(QueryRequestTest, ParseFromCommandDefaultBatchSize) { BSONObj cmdObj = fromjson("{find: 'testns'}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT(!qr->getBatchSize()); @@ -948,7 +949,7 @@ TEST(QueryRequestTest, ParseCommandIsFromFindCommand) { BSONObj cmdObj = fromjson("{find: 'testns'}"); const NamespaceString nss("test.testns"); bool isExplain = false; - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, isExplain))); ASSERT_FALSE(qr->getNToReturn()); @@ -982,7 +983,7 @@ TEST(QueryRequestTest, DefaultQueryParametersCorrect) { BSONObj cmdObj = fromjson("{find: 'testns'}"); const NamespaceString nss("test.testns"); - std::unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::makeFromFindCommand(nss, cmdObj, false))); ASSERT_FALSE(qr->getSkip()); @@ -1281,7 +1282,7 @@ TEST(QueryRequestTest, ParseFromLegacyObjMetaOpComment) { "{$query: {a: 1}," "$comment: {b: 2, c: {d: 'ParseFromLegacyObjMetaOpComment'}}}"); const NamespaceString nss("test.testns"); - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::fromLegacyQuery(nss, queryObj, BSONObj(), 0, 0, 0))); // Ensure that legacy comment meta-operator is parsed to a string comment @@ -1294,7 +1295,7 @@ TEST(QueryRequestTest, ParseFromLegacyStringMetaOpComment) { "{$query: {a: 1}," "$comment: 'ParseFromLegacyStringMetaOpComment'}"); const NamespaceString nss("test.testns"); - unique_ptr qr( + QueryRequest::UPtr qr( assertGet(QueryRequest::fromLegacyQuery(nss, queryObj, BSONObj(), 0, 0, 0))); ASSERT_EQ(qr->getComment(), "ParseFromLegacyStringMetaOpComment"); @@ -1315,7 +1316,7 @@ TEST(QueryRequestTest, ParseFromLegacyQuery) { $maxScan: 7 })"); const NamespaceString nss("test.testns"); - unique_ptr qr(assertGet(QueryRequest::fromLegacyQuery( + QueryRequest::UPtr qr(assertGet(QueryRequest::fromLegacyQuery( nss, queryObj, BSON("proj" << 1), kSkip, kNToReturn, QueryOption_Exhaust))); ASSERT_EQ(qr->nss(), nss); @@ -1344,7 +1345,7 @@ TEST(QueryRequestTest, ParseFromLegacyQueryUnwrapped) { foo: 1 })"); const NamespaceString nss("test.testns"); - unique_ptr qr(assertGet( + QueryRequest::UPtr qr(assertGet( QueryRequest::fromLegacyQuery(nss, queryObj, BSONObj(), 0, 0, QueryOption_Exhaust))); ASSERT_EQ(qr->nss(), nss); @@ -1367,12 +1368,15 @@ class QueryRequestTest : public ServiceContextTest {}; TEST_F(QueryRequestTest, ParseFromUUID) { auto opCtx = makeOperationContext(); + // UUIDCatalog registers recovery-unit changes; this parser test needs no real engine. + opCtx->setRecoveryUnit(new RecoveryUnitNoop(), + WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); // Register a UUID/Collection pair in the UUIDCatalog. const CollectionUUID uuid = UUID::gen(); const NamespaceString nss("test.testns"); - Collection coll(stdx::make_unique(nss)); + auto coll = stdx::make_unique(stdx::make_unique(nss)); UUIDCatalog& catalog = UUIDCatalog::get(opCtx.get()); - catalog.onCreateCollection(opCtx.get(), &coll, uuid); + catalog.onCreateCollection(opCtx.get(), std::move(coll), uuid); QueryRequest qr(uuid); // Ensure a call to refreshNSS succeeds. qr.refreshNSS(opCtx.get()); diff --git a/src/mongo/db/record_id.h b/src/mongo/db/record_id.h index fbd81241e1..421317ecd3 100644 --- a/src/mongo/db/record_id.h +++ b/src/mongo/db/record_id.h @@ -227,10 +227,9 @@ class RecordId { // repr() now just for Format::kLong int64_t repr() const { - // In the Eloq engine, the RecordId is equivalent to the "_id" field and is stored as a - // string. Therefore, the repr() function is meaningless. - // Returning 0 ensures compatibility with the existing API. - return 0; + // Preserve the legacy integer API for integer-backed stores and sentinels. + // Eloq's string-backed IDs have no integer representation. + return isLong() ? _getLongNoCheck() : 0; } /** diff --git a/src/mongo/db/record_id_test.cpp b/src/mongo/db/record_id_test.cpp index 389549bccc..965dcf094a 100644 --- a/src/mongo/db/record_id_test.cpp +++ b/src/mongo/db/record_id_test.cpp @@ -35,6 +35,18 @@ namespace mongo { namespace { +TEST(RecordId, IntegerRepresentationAndNullSentinel) { + ASSERT_EQUALS(42, RecordId(42).repr()); + ASSERT_EQUALS(-7, RecordId(-7).repr()); + ASSERT_FALSE(RecordId(42).isNull()); + ASSERT_TRUE(RecordId(0).isNull()); + ASSERT_TRUE(RecordId().isNull()); + RecordId stringId("key", 3); + ASSERT_TRUE(stringId.isStr()); + ASSERT_FALSE(stringId.isNull()); + ASSERT_EQUALS("key", stringId.getStringView()); +} + TEST(RecordId, HashEqual) { RecordId locA(1, 2); RecordId locB; diff --git a/src/mongo/db/repl/replication_coordinator_standalone.cpp b/src/mongo/db/repl/replication_coordinator_standalone.cpp new file mode 100644 index 0000000000..3a89d59274 --- /dev/null +++ b/src/mongo/db/repl/replication_coordinator_standalone.cpp @@ -0,0 +1,485 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kReplication + +#include "mongo/platform/basic.h" + +#include "mongo/db/repl/replication_coordinator_standalone.h" + +#include "mongo/db/global_settings.h" +#include "mongo/db/operation_context.h" +#include "mongo/db/repl/read_concern_args.h" +#include "mongo/db/repl/repl_set_config.h" +#include "mongo/db/storage/storage_engine.h" +#include "mongo/util/assert_util.h" + +namespace mongo { +namespace repl { + +namespace { + +// Only replication-only APIs without a Status result use this helper. APIs used by +// ordinary standalone commands must preserve their non-throwing Status contract. +MONGO_COMPILER_NORETURN void replicationUnavailable() { + uasserted(ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"); +} + +} // namespace + +ReplicationCoordinatorStandalone::ReplicationCoordinatorStandalone(ServiceContext* service) + : _service(service) {} + +ReplicationCoordinatorStandalone::~ReplicationCoordinatorStandalone() = default; + +void ReplicationCoordinatorStandalone::startup(OperationContext* opCtx) {} + +void ReplicationCoordinatorStandalone::shutdown(OperationContext* opCtx) {} + +const ReplSettings& ReplicationCoordinatorStandalone::getSettings() const { + return getGlobalReplSettings(); +} + +ReplicationCoordinator::Mode ReplicationCoordinatorStandalone::getReplicationMode() const { + return ReplicationCoordinator::Mode::modeNone; +} + +bool ReplicationCoordinatorStandalone::isMasterForReportingPurposes() { + return true; +} + +bool ReplicationCoordinatorStandalone::canAcceptWritesForDatabase(OperationContext* opCtx, + StringData dbName) { + return true; +} + +bool ReplicationCoordinatorStandalone::canAcceptWritesForDatabase_UNSAFE(OperationContext* opCtx, + StringData dbName) { + return true; +} + +bool ReplicationCoordinatorStandalone::canAcceptWritesFor(OperationContext* opCtx, + const NamespaceString& ns) { + return true; +} + +bool ReplicationCoordinatorStandalone::canAcceptWritesFor_UNSAFE(OperationContext* opCtx, + const NamespaceString& ns) { + return true; +} + +Status ReplicationCoordinatorStandalone::checkCanServeReadsFor(OperationContext* opCtx, + const NamespaceString& ns, + bool slaveOk) { + return Status::OK(); +} + +Status ReplicationCoordinatorStandalone::checkCanServeReadsFor_UNSAFE(OperationContext* opCtx, + const NamespaceString& ns, + bool slaveOk) { + return Status::OK(); +} + +bool ReplicationCoordinatorStandalone::isInPrimaryOrSecondaryState() const { + return false; +} + +bool ReplicationCoordinatorStandalone::shouldRelaxIndexConstraints(OperationContext* opCtx, + const NamespaceString& ns) { + return false; +} + +bool ReplicationCoordinatorStandalone::getMaintenanceMode() { + return false; +} + +WriteConcernOptions ReplicationCoordinatorStandalone::getGetLastErrorDefault() { + return WriteConcernOptions(); +} + +bool ReplicationCoordinatorStandalone::isReplEnabled() const { + return false; +} + +WriteConcernOptions ReplicationCoordinatorStandalone::populateUnsetWriteConcernOptionsSyncMode( + WriteConcernOptions wc) { + WriteConcernOptions writeConcern(wc); + if (writeConcern.syncMode == WriteConcernOptions::SyncMode::UNSET) { + writeConcern.syncMode = WriteConcernOptions::SyncMode::NONE; + } + return writeConcern; +} + +bool ReplicationCoordinatorStandalone::buildsIndexes() { + return true; +} + +OpTime ReplicationCoordinatorStandalone::getCurrentCommittedSnapshotOpTime() const { + return OpTime(); +} + +void ReplicationCoordinatorStandalone::appendDiagnosticBSON(mongo::BSONObjBuilder*) {} + +void ReplicationCoordinatorStandalone::appendConnectionStats( + executor::ConnectionPoolStats* stats) const {} + +MemberState ReplicationCoordinatorStandalone::getMemberState() const { + return MemberState::RS_STARTUP; +} + +Status ReplicationCoordinatorStandalone::waitForMemberState(MemberState, Milliseconds) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Seconds ReplicationCoordinatorStandalone::getSlaveDelaySecs() const { + return Seconds(0); +} + +void ReplicationCoordinatorStandalone::clearSyncSourceBlacklist() {} + +Status ReplicationCoordinatorStandalone::setFollowerMode(const MemberState&) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +ReplicationCoordinator::ApplierState ReplicationCoordinatorStandalone::getApplierState() { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::signalDrainComplete(OperationContext*, long long) { + replicationUnavailable(); +} + +Status ReplicationCoordinatorStandalone::waitForDrainFinish(Milliseconds) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +void ReplicationCoordinatorStandalone::signalUpstreamUpdater() { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::setMyHeartbeatMessage(const std::string&) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::setMyLastAppliedOpTimeForward(const OpTime&, + DataConsistency) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::setMyLastDurableOpTimeForward(const OpTime&) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::setMyLastAppliedOpTime(const OpTime&) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::setMyLastDurableOpTime(const OpTime&) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::resetMyLastOpTimes() { + replicationUnavailable(); +} + +OpTime ReplicationCoordinatorStandalone::getMyLastAppliedOpTime() const { + return OpTime(); +} + +OpTime ReplicationCoordinatorStandalone::getMyLastDurableOpTime() const { + return OpTime(); +} + +Status ReplicationCoordinatorStandalone::waitUntilOpTimeForRead( + OperationContext* opCtx, const ReadConcernArgs& readConcern) { + // Preserve the normal modeNone validation; storage capabilities belong to the engine. + const auto level = readConcern.getLevel(); + if (readConcern.getArgsAfterClusterTime() && level != ReadConcernLevel::kMajorityReadConcern && + level != ReadConcernLevel::kLocalReadConcern && + level != ReadConcernLevel::kSnapshotReadConcern) { + return {ErrorCodes::BadValue, + "Only readConcern level 'majority', 'local', or 'snapshot' is allowed when " + "specifying afterClusterTime"}; + } + if (readConcern.getArgsAtClusterTime() && level != ReadConcernLevel::kSnapshotReadConcern) { + return {ErrorCodes::BadValue, + "readConcern level 'snapshot' is required when specifying atClusterTime"}; + } + if (level == ReadConcernLevel::kMajorityReadConcern && + !opCtx->getServiceContext()->getStorageEngine()->supportsReadConcernMajority()) { + return {ErrorCodes::ReadConcernMajorityNotEnabled, + "Storage engine does not support majority read concern"}; + } + if (level == ReadConcernLevel::kSnapshotReadConcern && + !opCtx->getServiceContext()->getStorageEngine()->supportsReadConcernSnapshot()) { + return {ErrorCodes::InvalidOptions, + "Storage engine does not support snapshot read concern"}; + } + if (!readConcern.getArgsAfterClusterTime() && !readConcern.getArgsOpTime() && + !readConcern.getArgsAtClusterTime()) { + return Status::OK(); + } + return waitUntilOpTimeForReadUntil(opCtx, readConcern, boost::none); +} + +Status ReplicationCoordinatorStandalone::waitUntilOpTimeForReadUntil(OperationContext*, + const ReadConcernArgs&, + boost::optional) { + return {ErrorCodes::NotAReplicaSet, + "node needs to be a replica set member to use timestamped read concern"}; +} + +ReplicationCoordinator::StatusAndDuration ReplicationCoordinatorStandalone::awaitReplication( + OperationContext*, const OpTime&, const WriteConcernOptions&) { + return {Status::OK(), Milliseconds(0)}; +} + +Status ReplicationCoordinatorStandalone::stepDown(OperationContext*, + const bool, + const Milliseconds&, + const Milliseconds&) { + // The normal shutdown command always tries stepDown and explicitly accepts NotMaster. + return {ErrorCodes::NotMaster, "not running as a replica-set primary"}; +} + +OID ReplicationCoordinatorStandalone::getElectionId() { + return OID(); +} + +int ReplicationCoordinatorStandalone::getMyId() const { + return -1; +} + +Status ReplicationCoordinatorStandalone::resyncData(OperationContext*, bool) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +StatusWith ReplicationCoordinatorStandalone::prepareReplSetUpdatePositionCommand() const { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetGetStatus(BSONObjBuilder*, + ReplSetGetStatusResponseStyle) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +void ReplicationCoordinatorStandalone::fillIsMasterForReplSet(IsMasterResponse*) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::appendSlaveInfoData(BSONObjBuilder*) {} + +ReplSetConfig ReplicationCoordinatorStandalone::getConfig() const { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::processReplSetGetConfig(BSONObjBuilder*) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::processReplSetMetadata(const rpc::ReplSetMetadata&) {} + +void ReplicationCoordinatorStandalone::cancelAndRescheduleElectionTimeout() {} + +Status ReplicationCoordinatorStandalone::setMaintenanceMode(bool) { + // MaintenanceModeSetter tolerates a failed Status, but not an exception. + return {ErrorCodes::NoReplicationEnabled, + "can only set maintenance mode on replica set members"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetSyncFrom(OperationContext*, + const HostAndPort&, + BSONObjBuilder*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetFreeze(int, BSONObjBuilder*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processHeartbeat(const ReplSetHeartbeatArgs&, + ReplSetHeartbeatResponse*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetReconfig(OperationContext*, + const ReplSetReconfigArgs&, + BSONObjBuilder*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetInitiate(OperationContext*, + const BSONObj&, + BSONObjBuilder*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::abortCatchupIfNeeded() { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetFresh(const ReplSetFreshArgs&, + BSONObjBuilder*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetElect(const ReplSetElectArgs&, + BSONObjBuilder*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +Status ReplicationCoordinatorStandalone::processReplSetUpdatePosition(const UpdatePositionArgs&, + long long*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +std::vector ReplicationCoordinatorStandalone::getHostsWrittenTo(const OpTime&, bool) { + return {}; +} + +std::vector ReplicationCoordinatorStandalone::getOtherNodesInReplSet() const { + return {}; +} + +Status ReplicationCoordinatorStandalone::checkIfWriteConcernCanBeSatisfied( + const WriteConcernOptions&) const { + return {ErrorCodes::NoReplicationEnabled, "not running with --replSet"}; +} + +Status ReplicationCoordinatorStandalone::checkReplEnabledForCommand(BSONObjBuilder*) { + return Status(ErrorCodes::NoReplicationEnabled, "not running with --replSet"); +} + +HostAndPort ReplicationCoordinatorStandalone::chooseNewSyncSource(const OpTime&) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::blacklistSyncSource(const HostAndPort&, Date_t) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::resetLastOpTimesFromOplog(OperationContext*, + DataConsistency) { + replicationUnavailable(); +} + +bool ReplicationCoordinatorStandalone::shouldChangeSyncSource( + const HostAndPort&, const rpc::ReplSetMetadata&, boost::optional) { + replicationUnavailable(); +} + +void ReplicationCoordinatorStandalone::advanceCommitPoint(const OpTime&) { + replicationUnavailable(); +} + +OpTime ReplicationCoordinatorStandalone::getLastCommittedOpTime() const { + return OpTime(); +} + +Status ReplicationCoordinatorStandalone::processReplSetRequestVotes(OperationContext*, + const ReplSetRequestVotesArgs&, + ReplSetRequestVotesResponse*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +void ReplicationCoordinatorStandalone::prepareReplMetadata(const BSONObj&, + const OpTime&, + BSONObjBuilder*) const {} + +bool ReplicationCoordinatorStandalone::isV1ElectionProtocol() const { + return false; +} + +bool ReplicationCoordinatorStandalone::getWriteConcernMajorityShouldJournal() { + return false; +} + +Status ReplicationCoordinatorStandalone::processHeartbeatV1(const ReplSetHeartbeatArgsV1&, + ReplSetHeartbeatResponse*) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +void ReplicationCoordinatorStandalone::summarizeAsHtml(ReplSetHtmlSummary*) { + replicationUnavailable(); +} + +long long ReplicationCoordinatorStandalone::getTerm() { + return OpTime::kUninitializedTerm; +} + +Status ReplicationCoordinatorStandalone::updateTerm(OperationContext*, long long) { + return {ErrorCodes::BadValue, "cannot supply 'term' without active replication"}; +} + +void ReplicationCoordinatorStandalone::waitUntilSnapshotCommitted(OperationContext*, + const Timestamp&) { + replicationUnavailable(); +} + +size_t ReplicationCoordinatorStandalone::getNumUncommittedSnapshots() { + return 0; +} + +void ReplicationCoordinatorStandalone::dropAllSnapshots() {} + +Status ReplicationCoordinatorStandalone::stepUpIfEligible(bool skipDryRun) { + return {ErrorCodes::NoReplicationEnabled, + "MongoDB replication is not available in this standalone build"}; +} + +ReplSettings::IndexPrefetchConfig ReplicationCoordinatorStandalone::getIndexPrefetchConfig() const { + return _indexPrefetchConfig.load(); +} + +void ReplicationCoordinatorStandalone::setIndexPrefetchConfig( + const ReplSettings::IndexPrefetchConfig config) { + _indexPrefetchConfig.store(config); +} + +void ReplicationCoordinatorStandalone::signalDropPendingCollectionsRemovedFromStorage() {} + +} // namespace repl +} // namespace mongo diff --git a/src/mongo/db/repl/replication_coordinator_standalone.h b/src/mongo/db/repl/replication_coordinator_standalone.h new file mode 100644 index 0000000000..adf3f50133 --- /dev/null +++ b/src/mongo/db/repl/replication_coordinator_standalone.h @@ -0,0 +1,253 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include + +#include "mongo/db/repl/replication_coordinator.h" + +namespace mongo { +namespace repl { + +/** + * MongoDB standalone coordination only. Distributed transactions, replication and durability + * belong to the Eloq storage engine; this class must never start MongoDB replica-set workers. + */ +class ReplicationCoordinatorStandalone final : public repl::ReplicationCoordinator { + +public: + explicit ReplicationCoordinatorStandalone(ServiceContext* serviceContext); + ~ReplicationCoordinatorStandalone() override; + + ReplicationCoordinatorStandalone(const ReplicationCoordinatorStandalone&) = delete; + ReplicationCoordinatorStandalone& operator=(const ReplicationCoordinatorStandalone&) = delete; + + // Standalone implementations of the shared ReplicationCoordinator interface. + + void startup(OperationContext* opCtx) override; + + void shutdown(OperationContext* opCtx) override; + + // Returns the ServiceContext where this instance runs. + ServiceContext* getServiceContext() override { + return _service; + } + + const repl::ReplSettings& getSettings() const override; + + Mode getReplicationMode() const override; + bool getMaintenanceMode() override; + + bool isReplEnabled() const override; + bool isMasterForReportingPurposes() override; + bool isInPrimaryOrSecondaryState() const override; + + bool canAcceptWritesForDatabase(OperationContext* opCtx, StringData dbName) override; + bool canAcceptWritesForDatabase_UNSAFE(OperationContext* opCtx, StringData dbName) override; + + bool canAcceptWritesFor(OperationContext* opCtx, const NamespaceString& ns) override; + bool canAcceptWritesFor_UNSAFE(OperationContext* opCtx, const NamespaceString& ns) override; + + Status checkCanServeReadsFor(OperationContext* opCtx, + const NamespaceString& ns, + bool slaveOk) override; + Status checkCanServeReadsFor_UNSAFE(OperationContext* opCtx, + const NamespaceString& ns, + bool slaveOk) override; + + bool shouldRelaxIndexConstraints(OperationContext* opCtx, const NamespaceString& ns) override; + + WriteConcernOptions getGetLastErrorDefault() override; + + WriteConcernOptions populateUnsetWriteConcernOptionsSyncMode(WriteConcernOptions wc) override; + + bool buildsIndexes() override; + + // Replica-set operations reject requests; local server operations have modeNone semantics. + + + repl::MemberState getMemberState() const override; + + Status waitForMemberState(repl::MemberState, Milliseconds) override; + + Seconds getSlaveDelaySecs() const override; + + void clearSyncSourceBlacklist() override; + + repl::ReplicationCoordinator::StatusAndDuration awaitReplication( + OperationContext*, const repl::OpTime&, const WriteConcernOptions&) override; + + Status stepDown(OperationContext*, bool, const Milliseconds&, const Milliseconds&) override; + + Status checkIfWriteConcernCanBeSatisfied(const WriteConcernOptions&) const override; + + void setMyLastAppliedOpTime(const repl::OpTime&) override; + void setMyLastDurableOpTime(const repl::OpTime&) override; + + void setMyLastAppliedOpTimeForward(const repl::OpTime&, DataConsistency) override; + void setMyLastDurableOpTimeForward(const repl::OpTime&) override; + + void resetMyLastOpTimes() override; + + void setMyHeartbeatMessage(const std::string&) override; + + repl::OpTime getMyLastAppliedOpTime() const override; + repl::OpTime getMyLastDurableOpTime() const override; + + Status waitUntilOpTimeForReadUntil(OperationContext*, + const repl::ReadConcernArgs&, + boost::optional) override; + + Status waitUntilOpTimeForRead(OperationContext*, const repl::ReadConcernArgs&) override; + + OID getElectionId() override; + + int getMyId() const override; + + Status setFollowerMode(const repl::MemberState&) override; + + ApplierState getApplierState() override; + + void signalDrainComplete(OperationContext*, long long) override; + + Status waitForDrainFinish(Milliseconds) override; + + void signalUpstreamUpdater() override; + + Status resyncData(OperationContext*, bool) override; + + StatusWith prepareReplSetUpdatePositionCommand() const override; + + Status processReplSetGetStatus(BSONObjBuilder*, ReplSetGetStatusResponseStyle) override; + + void fillIsMasterForReplSet(repl::IsMasterResponse*) override; + + void appendSlaveInfoData(BSONObjBuilder*) override; + + repl::ReplSetConfig getConfig() const override; + + void processReplSetGetConfig(BSONObjBuilder*) override; + + void processReplSetMetadata(const rpc::ReplSetMetadata&) override; + + void advanceCommitPoint(const repl::OpTime& committedOpTime) override; + + void cancelAndRescheduleElectionTimeout() override; + + Status setMaintenanceMode(bool) override; + + Status processReplSetSyncFrom(OperationContext*, const HostAndPort&, BSONObjBuilder*) override; + + Status processReplSetFreeze(int, BSONObjBuilder*) override; + + Status processHeartbeat(const repl::ReplSetHeartbeatArgs&, + repl::ReplSetHeartbeatResponse*) override; + + Status processReplSetReconfig(OperationContext*, + const ReplSetReconfigArgs&, + BSONObjBuilder*) override; + + Status processReplSetInitiate(OperationContext*, const BSONObj&, BSONObjBuilder*) override; + + Status processReplSetFresh(const ReplSetFreshArgs&, BSONObjBuilder*) override; + + Status processReplSetElect(const ReplSetElectArgs& args, BSONObjBuilder* response) override; + + Status processReplSetUpdatePosition(const repl::UpdatePositionArgs&, long long*) override; + + std::vector getHostsWrittenTo(const repl::OpTime&, bool) override; + + std::vector getOtherNodesInReplSet() const override; + + Status checkReplEnabledForCommand(BSONObjBuilder*) override; + + + HostAndPort chooseNewSyncSource(const repl::OpTime&) override; + + void blacklistSyncSource(const HostAndPort&, Date_t) override; + + void resetLastOpTimesFromOplog(OperationContext*, DataConsistency) override; + + bool shouldChangeSyncSource(const HostAndPort&, + const rpc::ReplSetMetadata&, + boost::optional) override; + + repl::OpTime getLastCommittedOpTime() const override; + + Status processReplSetRequestVotes(OperationContext*, + const repl::ReplSetRequestVotesArgs&, + repl::ReplSetRequestVotesResponse*) override; + + void prepareReplMetadata(const BSONObj&, const repl::OpTime&, BSONObjBuilder*) const override; + + Status processHeartbeatV1(const repl::ReplSetHeartbeatArgsV1&, + repl::ReplSetHeartbeatResponse*) override; + + bool isV1ElectionProtocol() const override; + + bool getWriteConcernMajorityShouldJournal() override; + + void summarizeAsHtml(repl::ReplSetHtmlSummary*) override; + + void dropAllSnapshots() override; + + long long getTerm() override; + + Status updateTerm(OperationContext*, long long) override; + + repl::OpTime getCurrentCommittedSnapshotOpTime() const override; + + void waitUntilSnapshotCommitted(OperationContext*, const Timestamp&) override; + + void appendDiagnosticBSON(BSONObjBuilder*) override; + + void appendConnectionStats(executor::ConnectionPoolStats* stats) const override; + + size_t getNumUncommittedSnapshots() override; + + repl::ReplSettings::IndexPrefetchConfig getIndexPrefetchConfig() const override; + void setIndexPrefetchConfig(const repl::ReplSettings::IndexPrefetchConfig) override; + + Status stepUpIfEligible(bool skipDryRun) override; + + Status abortCatchupIfNeeded() override; + + void signalDropPendingCollectionsRemovedFromStorage() final; + +private: + // Back pointer to the ServiceContext that has started the instance. + ServiceContext* const _service; + + // Preserve the server parameter's set/get behavior without starting a prefetch worker. + std::atomic _indexPrefetchConfig{ + ReplSettings::IndexPrefetchConfig::PREFETCH_ALL}; +}; + +} // namespace repl +} // namespace mongo diff --git a/src/mongo/db/repl/replication_info.cpp b/src/mongo/db/repl/replication_info.cpp index ca56b59443..59b3bcde03 100644 --- a/src/mongo/db/repl/replication_info.cpp +++ b/src/mongo/db/repl/replication_info.cpp @@ -125,6 +125,8 @@ void appendReplicationInfo(OperationContext* opCtx, BSONObjBuilder& result, int t.done(); } +#ifndef ELOQDOC_STANDALONE + // Only the full server links the legacy remote replication-source client. if (level > 1) { invariant(!opCtx->lockState()->isLocked()); // note: there is no so-style timeout on this connection; perhaps we should have @@ -144,6 +146,7 @@ void appendReplicationInfo(OperationContext* opCtx, BSONObjBuilder& result, int } conn.done(); } +#endif sources.append(BSONObjBuilder::numStr(n++), bb.obj()); } diff --git a/src/mongo/db/server_options_server_helpers.cpp b/src/mongo/db/server_options_server_helpers.cpp index 464920f802..67f5d68c6e 100644 --- a/src/mongo/db/server_options_server_helpers.cpp +++ b/src/mongo/db/server_options_server_helpers.cpp @@ -133,8 +133,7 @@ Status addGeneralServerOptions(moe::OptionSection* options) { ->addOptionChaining("net.adaptiveThreadNum", "adaptiveThreadNum", moe::Int, - "set the thread num for adaptive service executor mode") - .setDefault(moe::Value(1)); + "set the thread num for adaptive service executor mode"); #if MONGO_ENTERPRISE_VERSION options->addOptionChaining("security.redactClientLogData", @@ -563,10 +562,11 @@ Status storeServerOptions(const moe::Environment& params) { return {ErrorCodes::BadValue, "adaptiveThreadNum can only be used with serviceExecutor=adaptive"}; } - serverGlobalParams.adaptiveThreadNum = params["net.adaptiveThreadNum"].as(); - if (serverGlobalParams.adaptiveThreadNum < 1) { + const auto threadCount = params["net.adaptiveThreadNum"].as(); + if (threadCount < 1) { return {ErrorCodes::BadValue, "adaptiveThreadNum has to be at least 1"}; } + serverGlobalParams.adaptiveThreadNum = threadCount; } if (params.count("security.transitionToAuth")) { diff --git a/src/mongo/db/server_options_test.cpp b/src/mongo/db/server_options_test.cpp index 8eddeb88df..97a7dcc105 100644 --- a/src/mongo/db/server_options_test.cpp +++ b/src/mongo/db/server_options_test.cpp @@ -85,6 +85,46 @@ class OptionsParserTester : public moe::OptionsParser { std::string _config; }; +TEST(ServiceExecutorOptions, ThreadCountHasNoImplicitCommandLineValue) { + OptionsParserTester parser; + moe::Environment environment; + moe::OptionSection options; + ASSERT_OK(::mongo::addGeneralServerOptions(&options)); + ASSERT_OK(parser.run(options, {"binaryname"}, {}, &environment)); + // The runtime defaults to one adaptive worker, but an omitted option must not be + // mistaken for an explicit request to configure an adaptive-only setting. + ASSERT_FALSE(environment.count("net.adaptiveThreadNum")); + ASSERT_OK(::mongo::storeServerOptions(environment)); +} + +TEST(ServiceExecutorOptions, ExplicitThreadCountRequiresAdaptiveExecutor) { + OptionsParserTester parser; + moe::Environment environment; + moe::OptionSection options; + ASSERT_OK(::mongo::addGeneralServerOptions(&options)); + ASSERT_OK(parser.run(options, {"binaryname", "--adaptiveThreadNum=2"}, {}, &environment)); + const auto status = ::mongo::storeServerOptions(environment); + ASSERT_EQ(status.code(), ErrorCodes::BadValue); + ASSERT_STRING_CONTAINS(status.reason(), "serviceExecutor=adaptive"); +} + +TEST(ServiceExecutorOptions, RejectsNonpositiveAdaptiveThreadCounts) { + const auto oldExecutor = ::mongo::serverGlobalParams.serviceExecutor; + const auto restore = ::mongo::MakeGuard( + [&] { ::mongo::serverGlobalParams.serviceExecutor = oldExecutor; }); + for (const auto* count : {"--adaptiveThreadNum=0", "--adaptiveThreadNum=-1"}) { + OptionsParserTester parser; + moe::Environment environment; + moe::OptionSection options; + ASSERT_OK(::mongo::addGeneralServerOptions(&options)); + ASSERT_OK(parser.run( + options, {"binaryname", "--serviceExecutor=adaptive", count}, {}, &environment)); + const auto status = ::mongo::storeServerOptions(environment); + ASSERT_EQ(status.code(), ErrorCodes::BadValue); + ASSERT_STRING_CONTAINS(status.reason(), "at least 1"); + } +} + TEST(Verbosity, Default) { OptionsParserTester parser; moe::Environment environment; diff --git a/src/mongo/db/service_context.cpp b/src/mongo/db/service_context.cpp index aa18be58f3..de67138cda 100644 --- a/src/mongo/db/service_context.cpp +++ b/src/mongo/db/service_context.cpp @@ -243,15 +243,8 @@ ServiceContext::UniqueOperationContext ServiceContext::makeOperationContext(Clie auto opCtx = ObjectPool::newObjectRawPointer(client, _nextOpId.fetchAndAdd(1)); onCreate(opCtx, _clientObservers); - // if (!opCtx->lockState()) { - // MONGO_UNREACHABLE; - // opCtx->setLockState(std::make_unique()); - // } - // if (!opCtx->recoveryUnit()) { - // MONGO_UNREACHABLE; - // opCtx->setRecoveryUnit(new RecoveryUnitNoop(), - // WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); - // } + // Storage observers own locker/recovery-unit setup. In particular, Eloq's early + // catalog initialization intentionally leaves the recovery unit unset. { stdx::lock_guard lk(*client); client->setOperationContext(opCtx); diff --git a/src/mongo/db/service_context_d_test_fixture.cpp b/src/mongo/db/service_context_d_test_fixture.cpp index 1965c44a2a..99df564227 100644 --- a/src/mongo/db/service_context_d_test_fixture.cpp +++ b/src/mongo/db/service_context_d_test_fixture.cpp @@ -36,6 +36,7 @@ #include "mongo/db/catalog/catalog_control.h" #include "mongo/db/catalog/uuid_catalog.h" #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/logical_clock.h" #include "mongo/db/op_observer_registry.h" #include "mongo/db/service_entry_point_mongod.h" @@ -46,6 +47,30 @@ #include "mongo/db/catalog/database_holder.h" namespace mongo { +namespace { + +// Engine-backed MongoDB fixtures need real local locking. Install it only in these +// fixtures, after the production observer has initialized the operation context. +class MongoDTestLockerObserver final : public ServiceContext::ClientObserver { +public: + void onCreateClient(Client*) override {} + void onDestroyClient(Client*) override {} + void onCreateOperationContext(OperationContext* opCtx) override { + // Preserve Eloq's early catalog-initialization path before an engine exists. + if (!opCtx->getServiceContext()->getStorageEngine()) { + return; + } + auto locker = std::make_unique(); + if (opCtx->lockState()) { + opCtx->swapLockState(std::move(locker)); + } else { + opCtx->setLockState(std::move(locker)); + } + } + void onDestroyOperationContext(OperationContext*) override {} +}; + +} // namespace ServiceContextMongoDTest::ServiceContextMongoDTest() : ServiceContextMongoDTest("ephemeralForTest") {} @@ -54,7 +79,8 @@ ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine) : ServiceContextMongoDTest(engine, RepairAction::kNoRepair) {} ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine, RepairAction repair) - : _tempDir("service_context_d_test_fixture") { + : ServiceContextTest(std::make_unique()), + _tempDir("service_context_d_test_fixture") { _stashedStorageParams.engine = std::exchange(storageGlobalParams.engine, std::move(engine)); _stashedStorageParams.engineSetByUser = diff --git a/src/mongo/db/service_context_test_fixture.cpp b/src/mongo/db/service_context_test_fixture.cpp index e9aba48280..e86fec8fe1 100644 --- a/src/mongo/db/service_context_test_fixture.cpp +++ b/src/mongo/db/service_context_test_fixture.cpp @@ -38,9 +38,16 @@ namespace mongo { -ScopedGlobalServiceContextForTest::ScopedGlobalServiceContextForTest() { +ScopedGlobalServiceContextForTest::ScopedGlobalServiceContextForTest() + : ScopedGlobalServiceContextForTest(nullptr) {} + +ScopedGlobalServiceContextForTest::ScopedGlobalServiceContextForTest( + std::unique_ptr observer) { setGlobalServiceContext(ServiceContext::make()); auto const serviceContext = getGlobalServiceContext(); + if (observer) { + serviceContext->registerClientObserver(std::move(observer)); + } auto observerRegistry = std::make_unique(); serviceContext->setOpObserver(std::move(observerRegistry)); } @@ -53,7 +60,10 @@ ServiceContext* ScopedGlobalServiceContextForTest::getServiceContext() { return getGlobalServiceContext(); } -ServiceContextTest::ServiceContextTest() { +ServiceContextTest::ServiceContextTest() : ServiceContextTest(nullptr) {} + +ServiceContextTest::ServiceContextTest(std::unique_ptr observer) + : ScopedGlobalServiceContextForTest(std::move(observer)) { Client::initThread(getThreadName()); } diff --git a/src/mongo/db/service_context_test_fixture.h b/src/mongo/db/service_context_test_fixture.h index 8588b1e18e..b520dd34a2 100644 --- a/src/mongo/db/service_context_test_fixture.h +++ b/src/mongo/db/service_context_test_fixture.h @@ -45,6 +45,8 @@ class ScopedGlobalServiceContextForTest { protected: ScopedGlobalServiceContextForTest(); + explicit ScopedGlobalServiceContextForTest( + std::unique_ptr observer); virtual ~ScopedGlobalServiceContextForTest(); }; @@ -64,6 +66,8 @@ class ServiceContextTest : public unittest::Test, public ScopedGlobalServiceCont protected: ServiceContextTest(); + // Opt-in fixture observer, registered before the first client is created. + explicit ServiceContextTest(std::unique_ptr observer); virtual ~ServiceContextTest(); }; diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp index 4aa5eac97c..73e59c5a0c 100644 --- a/src/mongo/db/service_entry_point_common.cpp +++ b/src/mongo/db/service_entry_point_common.cpp @@ -982,6 +982,7 @@ void execCommandDatabase(OperationContext* opCtx, throw; } } catch (const DBException& e) { +#ifndef ELOQDOC_STANDALONE // If we got a stale config, wait in case the operation is stuck in a critical section if (auto sce = e.extraInfo()) { if (!opCtx->getClient()->isInDirectClient()) { @@ -1003,6 +1004,7 @@ void execCommandDatabase(OperationContext* opCtx, .ignore(); } } +#endif // Append the error labels for transient transaction errors. auto errorLabels = getErrorLabels(sessionOptions, command->getName(), e.code()); @@ -1159,6 +1161,7 @@ DbResponse receivedQuery(OperationContext* opCtx, dbResponse.exhaustNS = runQuery(opCtx, q, nss, dbResponse.response); } catch (const AssertionException& e) { +#ifndef ELOQDOC_STANDALONE // If we got a stale config, wait in case the operation is stuck in a critical section if (auto sce = e.extraInfo()) { if (!opCtx->getClient()->isInDirectClient()) { @@ -1168,6 +1171,7 @@ DbResponse receivedQuery(OperationContext* opCtx, .ignore(); } } +#endif dbResponse.response.reset(); generateLegacyQueryErrorResponse(e, q, &op, &dbResponse.response); diff --git a/src/mongo/db/standalone_runtime_test.cpp b/src/mongo/db/standalone_runtime_test.cpp new file mode 100644 index 0000000000..45a3835725 --- /dev/null +++ b/src/mongo/db/standalone_runtime_test.cpp @@ -0,0 +1,129 @@ +#include "mongo/platform/basic.h" + +#include "mongo/db/global_settings.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/repl/read_concern_args.h" +#include "mongo/db/repl/replication_coordinator_standalone.h" +#include "mongo/unittest/unittest.h" + +namespace mongo { +namespace { + +TEST(StandaloneRuntime, UsesGlobalReplicationSettings) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_TRUE(&coordinator.getSettings() == &getGlobalReplSettings()); +} + +TEST(StandaloneRuntime, DoesNotAdvertiseMongoReplication) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_FALSE(coordinator.isReplEnabled()); + ASSERT_EQUALS(repl::ReplicationCoordinator::modeNone, coordinator.getReplicationMode()); + ASSERT_TRUE(coordinator.isMasterForReportingPurposes()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, + coordinator.checkReplEnabledForCommand(nullptr).code()); +} + +TEST(StandaloneRuntime, AcceptsLocalReadsAndWrites) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + NamespaceString ns("test.collection"); + ASSERT_TRUE(coordinator.canAcceptWritesFor(nullptr, ns)); + ASSERT_OK(coordinator.checkCanServeReadsFor(nullptr, ns, false)); + ASSERT_TRUE(coordinator.buildsIndexes()); + ASSERT_FALSE(coordinator.shouldRelaxIndexConstraints(nullptr, ns)); +} + +TEST(StandaloneRuntime, PreservesExplicitDurabilityRequest) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + WriteConcernOptions concern; + concern.syncMode = WriteConcernOptions::SyncMode::JOURNAL; + ASSERT_TRUE(WriteConcernOptions::SyncMode::JOURNAL == + coordinator.populateUnsetWriteConcernOptionsSyncMode(concern).syncMode); + concern.syncMode = WriteConcernOptions::SyncMode::FSYNC; + ASSERT_TRUE(WriteConcernOptions::SyncMode::FSYNC == + coordinator.populateUnsetWriteConcernOptionsSyncMode(concern).syncMode); + concern.syncMode = WriteConcernOptions::SyncMode::UNSET; + ASSERT_TRUE(WriteConcernOptions::SyncMode::NONE == + coordinator.populateUnsetWriteConcernOptionsSyncMode(concern).syncMode); +} + +TEST(StandaloneRuntime, DoesNotWaitForNonexistentReplicas) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_OK(coordinator.awaitReplication(nullptr, repl::OpTime(), WriteConcernOptions()).status); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, + coordinator.checkIfWriteConcernCanBeSatisfied(WriteConcernOptions()).code()); +} + +TEST(StandaloneRuntime, AllowsNormalShutdownCommandToSkipStepDown) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_EQUALS(ErrorCodes::NotMaster, + coordinator.stepDown(nullptr, false, Seconds(0), Seconds(120)).code()); +} + +TEST(StandaloneRuntime, MaintenanceModeReturnsStatusWithoutThrowing) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + // ServiceEntryPointCommon's MaintenanceModeSetter deliberately ignores these errors. + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, coordinator.setMaintenanceMode(true).code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, coordinator.setMaintenanceMode(false).code()); + ASSERT_FALSE(coordinator.getMaintenanceMode()); +} + +TEST(StandaloneRuntime, TermOnNonReplicationCommandsReturnsBadValue) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + // Both find and getMore validate a supplied term through this Status-returning API. + ASSERT_EQUALS(ErrorCodes::BadValue, coordinator.updateTerm(nullptr, 1).code()); + ASSERT_EQUALS(ErrorCodes::BadValue, + coordinator.updateTerm(nullptr, repl::OpTime::kUninitializedTerm).code()); + ASSERT_TRUE(coordinator.getTerm() == repl::OpTime::kUninitializedTerm); +} + +TEST(StandaloneRuntime, IndexPrefetchServerParameterRoundTripsWithoutReplication) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + using Config = repl::ReplSettings::IndexPrefetchConfig; + ASSERT_TRUE(coordinator.getIndexPrefetchConfig() == Config::PREFETCH_ALL); + for (auto config : {Config::PREFETCH_NONE, Config::PREFETCH_ID_ONLY, Config::PREFETCH_ALL}) { + coordinator.setIndexPrefetchConfig(config); + ASSERT_TRUE(coordinator.getIndexPrefetchConfig() == config); + } + ASSERT_FALSE(coordinator.isReplEnabled()); +} + +TEST(StandaloneRuntime, ReplicationOnlyStatusAPIsReturnErrorsWithoutThrowing) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, + coordinator.waitForMemberState(repl::MemberState::RS_SECONDARY, + Milliseconds(0)).code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, + coordinator.setFollowerMode(repl::MemberState::RS_SECONDARY).code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, + coordinator.waitForDrainFinish(Milliseconds(0)).code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, coordinator.resyncData(nullptr, false).code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, + coordinator.prepareReplSetUpdatePositionCommand().getStatus().code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, coordinator.abortCatchupIfNeeded().code()); + ASSERT_EQUALS(ErrorCodes::NoReplicationEnabled, coordinator.stepUpIfEligible(false).code()); +} + +TEST(StandaloneRuntime, ReplicationOnlyVoidAPIsReportUnavailable) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_THROWS_CODE(coordinator.signalUpstreamUpdater(), + DBException, + ErrorCodes::NoReplicationEnabled); +} + +TEST(StandaloneRuntime, AcceptsUntimestampedLocalReadConcern) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + ASSERT_OK(coordinator.waitUntilOpTimeForRead(nullptr, repl::ReadConcernArgs())); + ASSERT_OK(coordinator.waitUntilOpTimeForRead( + nullptr, repl::ReadConcernArgs(repl::ReadConcernLevel::kAvailableReadConcern))); +} + +TEST(StandaloneRuntime, RejectsReplicaSetTimestampWaits) { + repl::ReplicationCoordinatorStandalone coordinator(nullptr); + repl::ReadConcernArgs concern(repl::OpTime(Timestamp(1, 1), 1), + repl::ReadConcernLevel::kLocalReadConcern); + ASSERT_EQUALS(ErrorCodes::NotAReplicaSet, + coordinator.waitUntilOpTimeForRead(nullptr, concern).code()); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp index 3d212b0990..c8e5d5922f 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.cpp @@ -37,6 +37,7 @@ #include "mongo/db/catalog/index_catalog_entry.h" #include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_recovery_unit.h" #include "mongo/db/storage/index_entry_comparison.h" +#include "mongo/db/storage/record_store.h" #include "mongo/stdx/memory.h" #include "mongo/util/mongoutils/str.h" @@ -138,7 +139,10 @@ class EphemeralForTestBtreeBuilderImpl : public SortedDataBuilderInterface { class EphemeralForTestBtreeImpl : public SortedDataInterface { public: - EphemeralForTestBtreeImpl(IndexSet* data, bool isUnique) : _data(data), _isUnique(isUnique) { + EphemeralForTestBtreeImpl(IndexSet* data, + bool isUnique, + std::unique_ptr recordStore) + : _data(data), _isUnique(isUnique), _recordStore(std::move(recordStore)) { _currentKeySize = 0; } @@ -223,9 +227,11 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { class Cursor final : public SortedDataInterface::Cursor { public: - Cursor(OperationContext* opCtx, const IndexSet& data, bool isForward, bool isUnique) + Cursor(OperationContext* opCtx, const IndexSet& data, bool isForward, bool isUnique, + RecordStore* recordStore) : _opCtx(opCtx), _data(data), + _recordStore(recordStore), _forward(isForward), _isUnique(isUnique), _it(data.end()) {} @@ -242,7 +248,7 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { if (_isEOF) return {}; - return *_it; + return currentEntry(); } void setEndPosition(const BSONObj& key, bool inclusive) override { @@ -278,7 +284,7 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { : compareKeys(_it->key, query) > 0); } - return *_it; + return currentEntry(); } boost::optional seek(const IndexSeekPoint& seekPoint, @@ -290,7 +296,7 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { if (_isEOF) return {}; dassert(compareKeys(_it->key, query) >= 0); - return *_it; + return currentEntry(); } void save() override { @@ -352,6 +358,18 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { } private: + IndexKeyEntry currentEntry() const { + auto entry = *_it; + if (_recordStore) { + // Eloq's IDHack path consumes the document returned with the index entry. + // Standalone index harnesses have no record store and still return keys only. + auto record = _recordStore->dataFor(_opCtx, entry.loc); + record.makeOwned(); + entry.record = std::move(record); + } + return entry; + } + bool atEndPoint() const { return _endState && _it == _endState->it; } @@ -446,6 +464,7 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { OperationContext* _opCtx; // not owned const IndexSet& _data; + RecordStore* const _recordStore; const bool _forward; const bool _isUnique; bool _isEOF = true; @@ -472,7 +491,13 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { virtual std::unique_ptr newCursor(OperationContext* opCtx, bool isForward) const { - return stdx::make_unique(opCtx, *_data, isForward, _isUnique); + return stdx::make_unique(opCtx, *_data, isForward, _isUnique, _recordStore.get()); + } + + SortedDataInterface::Cursor::UPtr newCursorPtr(OperationContext* opCtx, + bool isForward) const override { + return {newCursor(opCtx, isForward).release(), + [](SortedDataInterface::Cursor* cursor) { delete cursor; }}; } virtual Status initAsEmpty(OperationContext* opCtx) { @@ -503,6 +528,7 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { IndexSet* _data; long long _currentKeySize; const bool _isUnique; + const std::unique_ptr _recordStore; }; } // namespace @@ -511,11 +537,19 @@ class EphemeralForTestBtreeImpl : public SortedDataInterface { SortedDataInterface* getEphemeralForTestBtreeImpl(const Ordering& ordering, bool isUnique, std::shared_ptr* dataInOut) { + return getEphemeralForTestBtreeImpl(ordering, isUnique, dataInOut, nullptr); +} + +SortedDataInterface* getEphemeralForTestBtreeImpl(const Ordering& ordering, + bool isUnique, + std::shared_ptr* dataInOut, + std::unique_ptr recordStore) { invariant(dataInOut); if (!*dataInOut) { *dataInOut = std::make_shared(IndexEntryComparison(ordering)); } - return new EphemeralForTestBtreeImpl(static_cast(dataInOut->get()), isUnique); + return new EphemeralForTestBtreeImpl( + static_cast(dataInOut->get()), isUnique, std::move(recordStore)); } } // namespace mongo diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.h index 9d44e3e80c..e719097176 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.h +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_btree_impl.h @@ -36,6 +36,7 @@ namespace mongo { class IndexCatalogEntry; +class RecordStore; /** * Caller takes ownership. @@ -45,4 +46,9 @@ SortedDataInterface* getEphemeralForTestBtreeImpl(const Ordering& ordering, bool isUnique, std::shared_ptr* dataInOut); +SortedDataInterface* getEphemeralForTestBtreeImpl(const Ordering& ordering, + bool isUnique, + std::shared_ptr* dataInOut, + std::unique_ptr recordStore); + } // namespace mongo diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp index 0bc0f4f841..64b135b1c6 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.cpp @@ -41,6 +41,73 @@ namespace mongo { +std::unique_ptr EphemeralForTestEngine::_getCatalogRecordStore() const { + stdx::lock_guard lk(_mutex); + auto it = _dataMap.find("_mdb_catalog"); + if (it == _dataMap.end() || !it->second) { + return nullptr; + } + auto data = it->second; + return stdx::make_unique("_mdb_catalog", &data); +} + +Status EphemeralForTestEngine::lockCollection(OperationContext* opCtx, + StringData ns, + bool isForWrite, + bool* exists, + std::string* version) { + auto catalog = _getCatalogRecordStore(); + RecordData record; + *exists = catalog && catalog->findRecord(opCtx, RecordId(ns.rawData(), ns.size()), &record); + version->clear(); + if (*exists) { + *version = record.toBson()["md"].Obj()["options"].Obj()["catalogVersion"].String(); + } + return Status::OK(); +} + +void EphemeralForTestEngine::listCollections(std::string_view dbName, + std::vector& out) const { + auto catalog = _getCatalogRecordStore(); + if (!catalog) { + return; + } + std::vector collections; + catalog->getAllCollections(collections); + for (const auto& ns : collections) { + if (NamespaceString(ns).db().toStringView() == dbName) { + out.push_back(ns); + } + } +} + +void EphemeralForTestEngine::listCollections(std::string_view dbName, + std::set& out) const { + std::vector collections; + listCollections(dbName, collections); + out.insert(collections.begin(), collections.end()); +} + +void EphemeralForTestEngine::listDatabases(std::vector& out) const { + auto catalog = _getCatalogRecordStore(); + if (!catalog) { + return; + } + std::vector collections; + catalog->getAllCollections(collections); + std::set databases; + for (const auto& ns : collections) { + databases.insert(NamespaceString(ns).db().toString()); + } + out.insert(out.end(), databases.begin(), databases.end()); +} + +bool EphemeralForTestEngine::databaseExists(std::string_view dbName) const { + std::vector collections; + listCollections(dbName, collections); + return !collections.empty(); +} + RecoveryUnit* EphemeralForTestEngine::newRecoveryUnit() { return new EphemeralForTestRecoveryUnit([this]() { stdx::lock_guard lk(_mutex); @@ -89,8 +156,11 @@ SortedDataInterface* EphemeralForTestEngine::getSortedDataInterface(OperationCon StringData ident, const IndexDescriptor* desc) { stdx::lock_guard lk(_mutex); + auto recordStore = stdx::make_unique( + desc->parentNS(), &_dataMap[desc->parentNS()]); return getEphemeralForTestBtreeImpl( - Ordering::make(desc->keyPattern()), desc->unique(), &_dataMap[ident]); + Ordering::make(desc->keyPattern()), desc->unique(), &_dataMap[ident], + std::move(recordStore)); } Status EphemeralForTestEngine::dropIdent(OperationContext* opCtx, StringData ident) { diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h index 38c9ea37ce..52406a1a9d 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h @@ -45,6 +45,23 @@ class EphemeralForTestEngine : public KVEngine { public: virtual RecoveryUnit* newRecoveryUnit(); + RecoveryUnit::UPtr newRecoveryUnitUPtr() override { + return {newRecoveryUnit(), [](RecoveryUnit* unit) { delete unit; }}; + } + + // Model the fork's namespace-keyed catalog in this in-memory test engine. Callers still + // use MongoDB's ordinary database/collection locks; there is no remote catalog cache. + Status lockCollection(OperationContext* opCtx, + StringData ns, + bool isForWrite, + bool* exists, + std::string* version) override; + void onAuthzDataChanged(OperationContext*) override {} + void listDatabases(std::vector& out) const override; + bool databaseExists(std::string_view dbName) const override; + void listCollections(std::string_view dbName, std::vector& out) const override; + void listCollections(std::string_view dbName, std::set& out) const override; + virtual Status createRecordStore(OperationContext* opCtx, StringData ns, StringData ident, @@ -115,6 +132,8 @@ class EphemeralForTestEngine : public KVEngine { } private: + std::unique_ptr _getCatalogRecordStore() const; + typedef StringMap> DataMap; mutable stdx::mutex _mutex; diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp index f2ed2f99de..9d985e0466 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.cpp @@ -343,6 +343,19 @@ void EphemeralForTestRecordStore::deleteRecord(OperationContext* opCtx, const Re deleteRecord_inlock(opCtx, loc); } +void EphemeralForTestRecordStore::getAllCollections(std::vector& collections) const { + if (ns() != "_mdb_catalog") { + return; + } + stdx::lock_guard lock(_data->recordsMutex); + for (const auto& entry : _data->records) { + auto name = entry.second.toRecordData().toBson()["ns"]; + if (name.type() == String) { + collections.push_back(name.String()); + } + } +} + void EphemeralForTestRecordStore::deleteRecord_inlock(OperationContext* opCtx, const RecordId& loc) { EphemeralForTestRecord* rec = recordFor(loc); @@ -410,7 +423,16 @@ StatusWith EphemeralForTestRecordStore::insertRecord( memcpy(rec.data.get(), data, len); RecordId loc; - if (_data->isOplog) { + if (ns() == "_mdb_catalog") { + // Eloq's KVCatalog looks up metadata by namespace, not by a numeric record location. + BSONObj metadata(data); + StringData key = metadata["isFeatureDoc"].trueValue() + ? "featureDocument"_sd : metadata["ns"].checkAndGetStringData(); + loc = RecordId(key.rawData(), key.size()); + if (_data->records.count(loc)) { + return {ErrorCodes::DuplicateKey, "catalog namespace already exists"}; + } + } else if (_data->isOplog) { StatusWith status = extractAndCheckLocForOplog(data, len); if (!status.isOK()) return status; diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.h index 5e0d7ba1e5..e705027fe0 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.h +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.h @@ -64,6 +64,15 @@ class EphemeralForTestRecordStore : public RecordStore { virtual bool findRecord(OperationContext* opCtx, const RecordId& loc, RecordData* rd) const; + bool findRecord(OperationContext* opCtx, + const RecordId& loc, + RecordData* rd, + bool isForWrite) const override { + return findRecord(opCtx, loc, rd); + } + + void getAllCollections(std::vector& collections) const override; + virtual void deleteRecord(OperationContext* opCtx, const RecordId& dl); virtual StatusWith insertRecord( diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store_test.cpp b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store_test.cpp index 602311e4ce..fef4fe16be 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store_test.cpp +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store_test.cpp @@ -31,14 +31,140 @@ #include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.h" #include "mongo/base/init.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/index_names.h" +#include "mongo/db/operation_context_noop.h" +#include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h" #include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_recovery_unit.h" #include "mongo/db/storage/record_store_test_harness.h" +#include "mongo/db/storage/sorted_data_interface.h" #include "mongo/stdx/memory.h" #include "mongo/unittest/unittest.h" namespace mongo { namespace { +class NamespaceCatalogTest : public unittest::Test { +public: + NamespaceCatalogTest() : opCtx(engine.newRecoveryUnit()) { + ASSERT_OK(engine.createRecordStore(&opCtx, "_mdb_catalog", "_mdb_catalog", {})); + catalog = engine.getRecordStore(&opCtx, "_mdb_catalog", "_mdb_catalog", {}); + } + + StatusWith insert(const BSONObj& doc) { + return catalog->insertRecord(&opCtx, doc.objdata(), doc.objsize(), Timestamp(), false); + } + + BSONObj metadata() { + return BSON("ns" << "test.collection" << "md" + << BSON("options" << BSON("catalogVersion" << "version-1"))); + } + + EphemeralForTestEngine engine; + OperationContextNoop opCtx; + std::unique_ptr catalog; +}; + +TEST_F(NamespaceCatalogTest, NamespaceKeysAndEnumerationFollowTransactionRollback) { + { + WriteUnitOfWork wuow(&opCtx); + auto id = insert(metadata()); + ASSERT_OK(id.getStatus()); + ASSERT_EQ(RecordId("test.collection", 15), id.getValue()); + ASSERT_TRUE(engine.databaseExists("test")); + // Roll back this insertion. + } + ASSERT_FALSE(engine.databaseExists("test")); + { + WriteUnitOfWork wuow(&opCtx); + ASSERT_OK(insert(metadata()).getStatus()); + wuow.commit(); + } + std::vector databases; + engine.listDatabases(databases); + ASSERT_EQ(1U, databases.size()); + ASSERT_EQ("test", databases.front()); + std::vector collections; + engine.listCollections("test", collections); + ASSERT_EQ(1U, collections.size()); + ASSERT_EQ("test.collection", collections.front()); + std::set names; + engine.listCollections("test", names); + ASSERT_EQ(1U, names.count("test.collection")); + ASSERT_FALSE(engine.databaseExists("other")); + + bool exists = false; + std::string version; + ASSERT_OK(engine.lockCollection(&opCtx, "test.collection", false, &exists, &version)); + ASSERT_TRUE(exists); + ASSERT_EQ("version-1", version); + ASSERT_OK(engine.lockCollection(&opCtx, "test.missing", true, &exists, &version)); + ASSERT_FALSE(exists); + ASSERT_TRUE(version.empty()); +} + +TEST_F(NamespaceCatalogTest, DuplicateInsertionAndAbortedDeletionPreserveMetadata) { + { + WriteUnitOfWork wuow(&opCtx); + ASSERT_OK(insert(metadata()).getStatus()); + wuow.commit(); + } + { + WriteUnitOfWork wuow(&opCtx); + ASSERT_EQ(ErrorCodes::DuplicateKey, insert(metadata()).getStatus().code()); + } + { + WriteUnitOfWork wuow(&opCtx); + catalog->deleteRecord(&opCtx, RecordId("test.collection", 15)); + ASSERT_FALSE(engine.databaseExists("test")); + } + ASSERT_TRUE(engine.databaseExists("test")); + { + WriteUnitOfWork wuow(&opCtx); + catalog->deleteRecord(&opCtx, RecordId("test.collection", 15)); + wuow.commit(); + } + ASSERT_FALSE(engine.databaseExists("test")); +} + +TEST_F(NamespaceCatalogTest, FeatureDocumentDoesNotAppearAsACollection) { + WriteUnitOfWork wuow(&opCtx); + auto id = insert(BSON("isFeatureDoc" << true << "ns" << BSONNULL)); + ASSERT_OK(id.getStatus()); + ASSERT_EQ(RecordId("featureDocument", 15), id.getValue()); + RecordData data; + ASSERT_TRUE(catalog->findRecord(&opCtx, id.getValue(), &data, true)); + std::vector collections; + catalog->getAllCollections(collections); + ASSERT_TRUE(collections.empty()); + wuow.commit(); +} + +TEST_F(NamespaceCatalogTest, PooledIndexCursorReturnsDocumentForIDHack) { + auto records = engine.getRecordStore(&opCtx, "test.collection", "test.collection", {}); + IndexDescriptor descriptor(nullptr, IndexNames::BTREE, + BSON("v" << 2 << "key" << BSON("_id" << 1) << "name" << "_id_" + << "ns" << "test.collection" << "unique" << true)); + ASSERT_OK(engine.createSortedDataInterface(&opCtx, "test.collection._id_", &descriptor)); + std::unique_ptr index( + engine.getSortedDataInterface(&opCtx, "test.collection._id_", &descriptor)); + BSONObj document = BSON("_id" << 42 << "payload" << "record from index lookup"); + { + WriteUnitOfWork wuow(&opCtx); + auto id = records->insertRecord(&opCtx, document.objdata(), document.objsize(), + Timestamp(), false); + ASSERT_OK(id.getStatus()); + ASSERT_OK(index->insert(&opCtx, BSON("" << 42), id.getValue(), false)); + wuow.commit(); + } + auto cursor = index->newCursorPtr(&opCtx); + auto entry = cursor->seekExact(BSON("" << 42)); + ASSERT_TRUE(entry); + ASSERT_TRUE(entry->record); + ASSERT_EQ(42, entry->record->toBson()["_id"].Int()); + ASSERT_EQ("record from index lookup", entry->record->toBson()["payload"].String()); +} + class EphemeralForTestHarnessHelper final : public RecordStoreHarnessHelper { public: EphemeralForTestHarnessHelper() {} diff --git a/src/mongo/db/storage/key_string_test.cpp b/src/mongo/db/storage/key_string_test.cpp index 0d87d0dfea..b0226e8121 100644 --- a/src/mongo/db/storage/key_string_test.cpp +++ b/src/mongo/db/storage/key_string_test.cpp @@ -103,8 +103,9 @@ TEST_F(KeyStringTest, Simple1) { ASSERT_BSONOBJ_LT(a, b); - ASSERT_LESS_THAN(KeyString(version, a, ALL_ASCENDING, RecordId()), - KeyString(version, b, ALL_ASCENDING, RecordId())); + // These keys use the integer RecordId format, including its zero sentinel. + ASSERT_LESS_THAN(KeyString(version, a, ALL_ASCENDING, RecordId(0)), + KeyString(version, b, ALL_ASCENDING, RecordId(0))); } #define ROUNDTRIP_ORDER(version, x, order) \ @@ -445,7 +446,7 @@ TEST_F(KeyStringTest, RecordIdOrder1) { KeyString a(version, BSON("" << 5), ordering, RecordId::min()); KeyString b(version, BSON("" << 5), ordering, RecordId(2)); KeyString c(version, BSON("" << 5), ordering, RecordId(3)); - KeyString d(version, BSON("" << 6), ordering, RecordId()); + KeyString d(version, BSON("" << 6), ordering, RecordId(0)); KeyString e(version, BSON("" << 6), ordering, RecordId(1)); ASSERT_LESS_THAN(a, b); @@ -1124,7 +1125,7 @@ TEST_F(KeyStringTest, RecordIds) { } if (rid.isNormal()) { - ASSERT_GT(ks, KeyString(version, RecordId())); + ASSERT_GT(ks, KeyString(version, RecordId(0))); ASSERT_GT(ks, KeyString(version, RecordId::min())); ASSERT_LT(ks, KeyString(version, RecordId::max())); diff --git a/src/mongo/db/storage/kv/SConscript b/src/mongo/db/storage/kv/SConscript index ea5b289b1e..786a502eef 100644 --- a/src/mongo/db/storage/kv/SConscript +++ b/src/mongo/db/storage/kv/SConscript @@ -134,8 +134,7 @@ env.CppUnitTest( '$BUILD_DIR/mongo/db/catalog/collection_options', '$BUILD_DIR/mongo/db/namespace_string', '$BUILD_DIR/mongo/db/service_context_test_fixture', - '$BUILD_DIR/mongo/db/storage/devnull/storage_devnull_core', - '$BUILD_DIR/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store', + '$BUILD_DIR/mongo/db/storage/ephemeral_for_test/storage_ephemeral_for_test_core', ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/auth/authmocks' diff --git a/src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp b/src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp index b27937839b..cd429f9610 100644 --- a/src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp +++ b/src/mongo/db/storage/kv/kv_collection_catalog_entry_test.cpp @@ -38,7 +38,7 @@ #include "mongo/db/index_names.h" #include "mongo/db/operation_context_noop.h" #include "mongo/db/service_context_test_fixture.h" -#include "mongo/db/storage/devnull/devnull_kv_engine.h" +#include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h" #include "mongo/db/storage/kv/kv_database_catalog_entry_mock.h" #include "mongo/db/storage/kv/kv_engine.h" #include "mongo/db/storage/kv/kv_storage_engine.h" @@ -51,31 +51,35 @@ namespace { class KVCollectionCatalogEntryTest : public ServiceContextTest { public: - KVCollectionCatalogEntryTest() - : _nss("unittests.kv_collection_catalog_entry"), - _storageEngine( - new DevNullKVEngine(), KVStorageEngineOptions(), kvDatabaseCatalogEntryMockFactory) { - _storageEngine.finishInit(); + KVCollectionCatalogEntryTest() : _nss("unittests.kv_collection_catalog_entry") { + auto engine = std::make_unique( + new EphemeralForTestEngine(), KVStorageEngineOptions(), kvDatabaseCatalogEntryMockFactory); + _storageEngine = engine.get(); + getServiceContext()->setStorageEngine(std::move(engine)); + _storageEngine->finishInit(); } ~KVCollectionCatalogEntryTest() { - _storageEngine.cleanShutdown(); + _storageEngine->cleanShutdown(); } std::unique_ptr newOperationContext() { - return stdx::make_unique(_storageEngine.newRecoveryUnit()); + auto opCtx = stdx::make_unique(getClient(), 0); + opCtx->setRecoveryUnit(_storageEngine->newRecoveryUnit(), + WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); + return opCtx; } void setUp() final { auto opCtx = newOperationContext(); DatabaseCatalogEntry* dbEntry = - _storageEngine.getDatabaseCatalogEntry(opCtx.get(), _nss.db()); + _storageEngine->getDatabaseCatalogEntry(opCtx.get(), _nss.db()); { WriteUnitOfWork wuow(opCtx.get()); - const bool allocateDefaultSpace = true; - ASSERT_OK(dbEntry->createCollection( - opCtx.get(), _nss.ns(), CollectionOptions(), allocateDefaultSpace)); + auto idIndex = BSON("v" << 2 << "key" << BSON("_id" << 1) << "name" << "_id_" + << "ns" << _nss.ns()); + ASSERT_OK(dbEntry->createCollection(opCtx.get(), _nss, CollectionOptions(), idIndex)); wuow.commit(); } } @@ -83,8 +87,8 @@ class KVCollectionCatalogEntryTest : public ServiceContextTest { CollectionCatalogEntry* getCollectionCatalogEntry() { auto opCtx = newOperationContext(); DatabaseCatalogEntry* dbEntry = - _storageEngine.getDatabaseCatalogEntry(opCtx.get(), _nss.db()); - return dbEntry->getCollectionCatalogEntry(_nss.ns()); + _storageEngine->getDatabaseCatalogEntry(opCtx.get(), _nss.db()); + return dbEntry->getCollectionCatalogEntry(opCtx.get(), _nss.ns()); } std::string createIndex(BSONObj keyPattern, std::string indexType = IndexNames::BTREE) { @@ -136,7 +140,7 @@ class KVCollectionCatalogEntryTest : public ServiceContextTest { } const NamespaceString _nss; - KVStorageEngine _storageEngine; + KVStorageEngine* _storageEngine; size_t numIndexesCreated = 0; }; diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp index d9623f23d9..bb2bdd3a75 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp @@ -279,11 +279,11 @@ void KVStorageEngine::closeCatalog(OperationContext* opCtx) { _dumpCatalog(opCtx); } - // stdx::lock_guard lock(_dbsLock); - // for (auto entry : _dbMapVector) { - // delete entry.second; - // } - _dbMapVector.clear(); + // The slots are indexed by LocalThread::ID() when the catalog is reopened. + // Clear their entries, not the slots themselves. + for (auto& dbMap : _dbMapVector) { + dbMap.clear(); + } _catalog.reset(nullptr); _catalogRecordStore.reset(nullptr); diff --git a/src/mongo/db/storage/storage_engine_init.cpp b/src/mongo/db/storage/storage_engine_init.cpp index fc31b4e44c..92e4e267ad 100644 --- a/src/mongo/db/storage/storage_engine_init.cpp +++ b/src/mongo/db/storage/storage_engine_init.cpp @@ -36,7 +36,6 @@ #include "mongo/base/init.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/concurrency/eloq_locker_noop.h" -#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/operation_context.h" #include "mongo/db/storage/storage_engine_lock_file.h" #include "mongo/db/storage/storage_engine_metadata.h" @@ -336,8 +335,7 @@ class StorageClientObserver final : public ServiceContext::ClientObserver { void onDestroyClient(Client* client) override{}; void onCreateOperationContext(OperationContext* opCtx) override { auto service = opCtx->getServiceContext(); - // Eloq TableSchema require a OperationContext while storageEngine is not set. - // Acutally, we always use EloqLockerNoop. + // Eloq TableSchema needs an OperationContext even before its engine is installed. auto storageEngine = service->getStorageEngine(); // NOTE(schwerin): The following uassert would be more desirable than the early return when // no storage engine is set, but to achieve that we would have to ensure that this file was @@ -348,10 +346,17 @@ class StorageClientObserver final : public ServiceContext::ClientObserver { // storageEngine); - if (opCtx->lockState()) { + if (dynamic_cast(opCtx->lockState())) { opCtx->resetLockState(); } else { - opCtx->setLockState(stdx::make_unique()); + // Production always uses Eloq locking. A pooled context may carry a test + // fixture's locker; replace it rather than keeping it in a server context. + auto locker = stdx::make_unique(); + if (opCtx->lockState()) { + opCtx->swapLockState(std::move(locker)); + } else { + opCtx->setLockState(std::move(locker)); + } } if (storageEngine) { diff --git a/src/mongo/db/storage/test_harness_helper.h b/src/mongo/db/storage/test_harness_helper.h index cd21223be1..2fcb909ec9 100644 --- a/src/mongo/db/storage/test_harness_helper.h +++ b/src/mongo/db/storage/test_harness_helper.h @@ -32,6 +32,7 @@ #include #include +#include "mongo/db/concurrency/locker_noop.h" #include "mongo/db/jsobj.h" #include "mongo/db/operation_context_noop.h" #include "mongo/db/record_id.h" @@ -50,6 +51,11 @@ class HarnessHelper { virtual ServiceContext::UniqueOperationContext newOperationContext(Client* const client) { auto opCtx = client->makeOperationContext(); + // Engine harnesses supply their own recovery units, without installing a server + // storage observer. WriteUnitOfWork still needs a locker on fresh/pooled contexts. + if (!opCtx->lockState()) { + opCtx->setLockState(stdx::make_unique()); + } opCtx->setRecoveryUnit(newRecoveryUnit().release(), WriteUnitOfWork::RecoveryUnitState::kNotInUnitOfWork); return opCtx; diff --git a/src/mongo/db/ttl.cpp b/src/mongo/db/ttl.cpp index 548bfcbdb1..3d390c73ee 100644 --- a/src/mongo/db/ttl.cpp +++ b/src/mongo/db/ttl.cpp @@ -48,6 +48,7 @@ #include "mongo/db/commands/fsync_locked.h" #include "mongo/db/commands/server_status_metric.h" #include "mongo/db/concurrency/write_conflict_exception.h" +#include "mongo/db/coro_sync.h" #include "mongo/db/db_raii.h" #include "mongo/db/exec/delete.h" #include "mongo/db/index/index_descriptor.h" @@ -60,6 +61,7 @@ #include "mongo/util/background.h" #include "mongo/util/concurrency/idle_thread_block.h" #include "mongo/util/exit.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" namespace mongo { @@ -73,6 +75,7 @@ ServerStatusMetricField ttlDeletedDocumentsDisplay("ttl.deletedDocume MONGO_EXPORT_SERVER_PARAMETER(ttlMonitorEnabled, bool, true); MONGO_EXPORT_SERVER_PARAMETER(ttlMonitorSleepSecs, int, 60); // used for testing +MONGO_FAIL_POINT_DEFINE(hangTTLMonitorBeforeStorageAccess); class TTLMonitor : public BackgroundJob { public: @@ -85,15 +88,42 @@ class TTLMonitor : public BackgroundJob { static std::string secondsExpireField; + void shutdown() { + { + stdx::unique_lock lock(_mutex); + _shutdownRequested = true; + _cond.notify_all(); + // Shutdown can run on an Eloq request coroutine. Yield that coroutine while an + // active TTL pass finishes; blocking its worker could strand the pass's requests. + _cond.wait(lock, [this] { return _finished; }); + } + // run() has released its Client and storage resources. BackgroundJob only has its + // completion bookkeeping left, so this native wait cannot block a Substrate request. + wait(); + } + virtual void run() { + ON_BLOCK_EXIT([this] { + stdx::lock_guard lock(_mutex); + _finished = true; + _cond.notify_all(); + }); Client::initThread(name().c_str()); + // Destroy the Client before publishing completion to the shutdown coroutine. ON_BLOCK_EXIT([] { Client::destroy(); }); AuthorizationSession::get(cc())->grantInternalAuthorization(); while (!globalInShutdownDeprecated()) { { + stdx::unique_lock lock(_mutex); MONGO_IDLE_THREAD_BLOCK; - sleepsecs(ttlMonitorSleepSecs.load()); + const auto interval = Seconds(ttlMonitorSleepSecs.load()); + LOG(1) << "TTL monitor waiting for next pass: " << interval.count() << " seconds"; + if (_cond.wait_for(lock, interval.toSystemDuration(), + [this] { return _shutdownRequested; }) || + globalInShutdownDeprecated()) { + break; + } } LOG(3) << "thread awake"; @@ -119,6 +149,11 @@ class TTLMonitor : public BackgroundJob { } private: + coro::Mutex _mutex; + coro::ConditionVariable _cond; + bool _shutdownRequested = false; + bool _finished = false; + void doTTLPass() { const ServiceContext::UniqueOperationContext opCtxPtr = cc().makeOperationContext(); OperationContext& opCtx = *opCtxPtr; @@ -129,6 +164,18 @@ class TTLMonitor : public BackgroundJob { !repl::ReplicationCoordinator::get(&opCtx)->getMemberState().readable()) return; + // Exercise shutdown with a pass already committed to accessing storage. Releasing + // on the global shutdown flag lets the shutdown command drain it without another + // client connection (listening sockets are already closed during shutdown). + if (MONGO_FAIL_POINT(hangTTLMonitorBeforeStorageAccess)) { + log() << "TTL monitor paused before storage access"; + while (MONGO_FAIL_POINT(hangTTLMonitorBeforeStorageAccess) && + !globalInShutdownDeprecated()) { + sleepmillis(10); + } + log() << "TTL monitor resuming storage access"; + } + // EloqDoc: TTLCollectionCache is unreliable. // // TTLCollectionCache& ttlCollectionCache = @@ -317,15 +364,37 @@ class TTLMonitor : public BackgroundJob { }; namespace { -// The global TTLMonitor object is intentionally leaked. Even though it is only used in one -// function, we declare it here to indicate to the leak sanitizer that the leak of this object -// should not be reported. +// Keep the monitor alive for the process lifetime so repeated shutdown calls can safely wait +// for it. The global pointer also makes this intentional lifetime visible to leak sanitizer. TTLMonitor* ttlMonitor = nullptr; +stdx::mutex ttlMonitorMutex; +bool ttlShutdownRequested = false; } // namespace void startTTLBackgroundJob() { - ttlMonitor = new TTLMonitor(); - ttlMonitor->go(); + stdx::lock_guard lock(ttlMonitorMutex); + if (ttlShutdownRequested) + return; + invariant(!ttlMonitor); + auto monitor = std::make_unique(); + monitor->go(); + ttlMonitor = monitor.release(); +} + +void shutdownTTLBackgroundJob() { + TTLMonitor* monitor; + { + // Shutdown may race startup or run before TTL is initialized. Do not allow a worker + // to be published after shutdown has observed that no worker exists. + stdx::lock_guard lock(ttlMonitorMutex); + ttlShutdownRequested = true; + monitor = ttlMonitor; + } + if (monitor) { + log() << "Stopping TTL monitor"; + monitor->shutdown(); + log() << "TTL monitor stopped"; + } } std::string TTLMonitor::secondsExpireField = "expireAfterSeconds"; diff --git a/src/mongo/db/ttl.h b/src/mongo/db/ttl.h index c015f394ce..f8ec01639c 100644 --- a/src/mongo/db/ttl.h +++ b/src/mongo/db/ttl.h @@ -32,4 +32,6 @@ namespace mongo { void startTTLBackgroundJob(); +// Wake and drain the TTL worker before storage teardown. Safe if startup never started TTL. +void shutdownTTLBackgroundJob(); } diff --git a/src/mongo/db/update/update_driver_test.cpp b/src/mongo/db/update/update_driver_test.cpp index cfa3241ce1..cbbdcaf48c 100644 --- a/src/mongo/db/update/update_driver_test.cpp +++ b/src/mongo/db/update/update_driver_test.cpp @@ -169,10 +169,10 @@ class CreateFromQueryFixture : public mongo::unittest::Test { : _opCtx(_serviceContext.makeOperationContext()), // _driverOps(new UpdateDriver(new ExpressionContext(_opCtx.get(), nullptr))), _driverOps(new UpdateDriver( - ObjectPool::newObjectRawPointer(_opCtx, nullptr))), + ObjectPool::newObjectRawPointer(_opCtx.get(), nullptr))), // _driverRepl(new UpdateDriver(new ExpressionContext(_opCtx.get(), nullptr))) _driverRepl(new UpdateDriver( - ObjectPool::newObjectRawPointer(_opCtx, nullptr))) { + ObjectPool::newObjectRawPointer(_opCtx.get(), nullptr))) { _driverOps->parse(fromjson("{$set:{'_':1}}"), _arrayFilters).transitional_ignore(); _driverRepl->parse(fromjson("{}"), _arrayFilters).transitional_ignore(); } diff --git a/src/mongo/platform/stack_locator.cpp b/src/mongo/platform/stack_locator.cpp index 3668615611..e937ba5783 100644 --- a/src/mongo/platform/stack_locator.cpp +++ b/src/mongo/platform/stack_locator.cpp @@ -30,30 +30,25 @@ #include "mongo/platform/stack_locator.h" +#include #include "mongo/util/assert_util.h" namespace mongo { boost::optional StackLocator::available() const { - // Returns a disengaged optional since the remaining stack cannot be determined. - return boost::none; - if (!begin() || !end()) return boost::none; - // Technically, it is undefined behavior to compare or subtract - // two pointers that do not point into the same - // aggregate. However, we know that these are both pointers within - // the same stack, and it seems unlikely that the compiler will - // see that it can elide the comparison here. - - const auto cbegin = reinterpret_cast(begin()); - const auto cthis = reinterpret_cast(this); - const auto cend = reinterpret_cast(end()); + // Compare addresses as integers: coroutine and native stack pointers need not + // point into the same allocation. + const auto cbegin = reinterpret_cast(begin()); + const auto cthis = reinterpret_cast(this); + const auto cend = reinterpret_cast(end()); - // TODO: Assumes that stack grows downward - invariant(cthis <= cbegin); - invariant(cthis > cend); + // A coroutine may be running on a different stack from the OS thread whose + // bounds we queried. Do not report (or assert on) those unrelated bounds. + if (cthis > cbegin || cthis <= cend) + return boost::none; std::size_t avail = cthis - cend; diff --git a/src/mongo/s/shard_id_test.cpp b/src/mongo/s/shard_id_test.cpp index 6393c9e9a9..2914ce6e10 100644 --- a/src/mongo/s/shard_id_test.cpp +++ b/src/mongo/s/shard_id_test.cpp @@ -86,8 +86,11 @@ TEST(ShardId, Compare) { ShardId sa1(a1); ShardId sb(b); ASSERT_EQUALS(sa.compare(sa1), a.compare(a1)); - ASSERT_EQUALS(sb.compare(sa1), b.compare(a1)); - ASSERT_EQUALS(sa.compare(sb), a.compare(b)); + // String comparisons promise the sign, not a particular negative/positive magnitude. + ASSERT_EQUALS(sb.compare(sa1) < 0, b.compare(a1) < 0); + ASSERT_EQUALS(sb.compare(sa1) > 0, b.compare(a1) > 0); + ASSERT_EQUALS(sa.compare(sb) < 0, a.compare(b) < 0); + ASSERT_EQUALS(sa.compare(sb) > 0, a.compare(b) > 0); } TEST(ShardId, Equals) { diff --git a/src/mongo/transport/service_executor_adaptive.cpp b/src/mongo/transport/service_executor_adaptive.cpp index 5906a1b3cb..7b65f077c3 100644 --- a/src/mongo/transport/service_executor_adaptive.cpp +++ b/src/mongo/transport/service_executor_adaptive.cpp @@ -188,6 +188,9 @@ ServiceExecutorAdaptive::~ServiceExecutorAdaptive() { Status ServiceExecutorAdaptive::start() { invariant(!_isRunning.load()); + if (_reactorHandles.empty()) { + return {ErrorCodes::InvalidOptions, "Adaptive executor requires at least one reactor"}; + } _isRunning.store(true); // _controllerThread = stdx::thread(&ServiceExecutorAdaptive::_controllerThreadRoutine, this); for (auto i = 0; i < _reactorHandles.size(); i++) { @@ -204,7 +207,9 @@ Status ServiceExecutorAdaptive::shutdown(Milliseconds timeout) { _isRunning.store(false); _scheduleCondition.notify_one(); - _controllerThread.join(); + if (_controllerThread.joinable()) { + _controllerThread.join(); + } stdx::unique_lock lk(_threadsMutex); for (auto& reactorHandle : _reactorHandles) { @@ -222,78 +227,9 @@ Status ServiceExecutorAdaptive::shutdown(Milliseconds timeout) { Status ServiceExecutorAdaptive::schedule(ServiceExecutorAdaptive::Task task, ScheduleFlags flags, ServiceExecutorTaskName taskName) { + // EloqDoc uses this executor only to run ingress reactors. Session tasks are + // assigned to ServiceExecutorCoroutine by ServiceEntryPointImpl::startSession(). MONGO_UNREACHABLE; - auto scheduleTime = _tickSource->getTicks(); - auto pendingCounterPtr = (flags & kDeferredTask) ? &_deferredTasksQueued : &_tasksQueued; - pendingCounterPtr->addAndFetch(1); - - if (!_isRunning.load()) { - return {ErrorCodes::ShutdownInProgress, "Executor is not running"}; - } - - auto wrappedTask = - [this, task = std::move(task), scheduleTime, pendingCounterPtr, taskName, flags] { - pendingCounterPtr->subtractAndFetch(1); - auto start = _tickSource->getTicks(); - _totalSpentQueued.addAndFetch(start - scheduleTime); - - _localThreadState->threadMetrics[static_cast(taskName)] - ._totalSpentQueued.addAndFetch(start - scheduleTime); - - if (_localThreadState->recursionDepth++ == 0) { - _localThreadState->executing.markRunning(); - _threadsInUse.addAndFetch(1); - } - const auto guard = MakeGuard([this, taskName] { - if (--_localThreadState->recursionDepth == 0) { - _localThreadState->executingCurRun += - _localThreadState->executing.markStopped(); - _threadsInUse.subtractAndFetch(1); - } - _totalExecuted.addAndFetch(1); - _localThreadState->threadMetrics[static_cast(taskName)] - ._totalExecuted.addAndFetch(1); - }); - - TickTimer _localTimer(_tickSource); - task(); - _localThreadState->threadMetrics[static_cast(taskName)] - ._totalSpentExecuting.addAndFetch(_localTimer.sinceStartTicks()); - - if ((flags & ServiceExecutor::kMayYieldBeforeSchedule) && - (_localThreadState->markIdleCounter++ & 0xf)) { - markThreadIdle(); - } - }; - - // Dispatching a task on the io_context will run the task immediately, and may run it - // on the current thread (if the current thread is running the io_context right now). - // - // Posting a task on the io_context will run the task without recursion. - // - // If the task is allowed to recurse and we are not over the depth limit, dispatch it so it - // can be called immediately and recursively. - if ((flags & kMayRecurse) && - (_localThreadState->recursionDepth + 1 < _config->recursionLimit())) { - // _reactorHandle->schedule(Reactor::kDispatch, std::move(wrappedTask)); - } else { - // _reactorHandle->schedule(Reactor::kPost, std::move(wrappedTask)); - } - - _lastScheduleTimer.reset(); - _totalQueued.addAndFetch(1); - - _accumulatedMetrics[static_cast(taskName)]._totalQueued.addAndFetch(1); - - // Deferred tasks never count against the thread starvation avoidance. For other tasks, we - // notify the controller thread that a task has been scheduled and we should monitor thread - // starvation. - if (_isStarved() && !(flags & kDeferredTask)) { - _starvationCheckRequests.addAndFetch(1); - _scheduleCondition.notify_one(); - } - - return Status::OK(); } bool ServiceExecutorAdaptive::_isStarved() const { diff --git a/src/mongo/transport/service_executor_adaptive.h b/src/mongo/transport/service_executor_adaptive.h index 35ba172e2b..ea13114d37 100644 --- a/src/mongo/transport/service_executor_adaptive.h +++ b/src/mongo/transport/service_executor_adaptive.h @@ -185,7 +185,7 @@ class ServiceExecutorAdaptive : public ServiceExecutor { ThreadState(TickSource* ts) : running(ts), executing(ts) {} CumulativeTickTimer running; - TickSource::Tick executingCurRun; + TickSource::Tick executingCurRun = 0; CumulativeTickTimer executing; MetricsArray threadMetrics; std::int64_t markIdleCounter = 0; diff --git a/src/mongo/transport/service_executor_test.cpp b/src/mongo/transport/service_executor_test.cpp index da2f6a867d..49a67e75c2 100644 --- a/src/mongo/transport/service_executor_test.cpp +++ b/src/mongo/transport/service_executor_test.cpp @@ -32,16 +32,27 @@ #include "boost/optional.hpp" +#include "mongo/db/local_thread_state.h" #include "mongo/db/service_context.h" +#include "mongo/stdx/future.h" #include "mongo/transport/service_executor_adaptive.h" +#include "mongo/transport/service_executor_coroutine.h" #include "mongo/transport/service_executor_synchronous.h" #include "mongo/transport/service_executor_task_names.h" +#include "mongo/unittest/death_test.h" #include "mongo/unittest/unittest.h" #include "mongo/util/log.h" #include "mongo/util/scopeguard.h" #include +#ifdef ELOQ_MODULE_ENABLED +#include +#include + +DECLARE_bool(brpc_worker_as_ext_processor); +#endif + namespace mongo { namespace { using namespace transport; @@ -92,7 +103,8 @@ class ASIOReactor : public transport::Reactor { ASIOReactor() : _ioContext() {} void run() noexcept final { - MONGO_UNREACHABLE; + asio::io_context::work work(_ioContext); + _ioContext.run(); } void runFor(Milliseconds time) noexcept final { @@ -113,7 +125,7 @@ class ASIOReactor : public transport::Reactor { void drain() override final { _ioContext.restart(); while (_ioContext.poll()) { - LOG(1) << "Draining remaining work in reactor."; + MONGO_LOG(1) << "Draining remaining work in reactor."; } _ioContext.stop(); } @@ -152,15 +164,20 @@ class ServiceExecutorAdaptiveFixture : public unittest::Test { auto scOwned = ServiceContext::make(); setGlobalServiceContext(std::move(scOwned)); - auto configOwned = stdx::make_unique(); - executorConfig = configOwned.get(); + reactors = {std::make_shared(), std::make_shared()}; executor = stdx::make_unique( - getGlobalServiceContext(), std::make_shared(), std::move(configOwned)); + getGlobalServiceContext(), std::vector(reactors), + stdx::make_unique()); + } + + void tearDown() override { + ASSERT_OK(executor->shutdown(kShutdownTime)); + executor.reset(); + setGlobalServiceContext({}); } - ServiceExecutorAdaptive::Options* executorConfig; + std::vector reactors; std::unique_ptr executor; - std::shared_ptr asioIOCtx; }; class ServiceExecutorSynchronousFixture : public unittest::Test { @@ -176,34 +193,130 @@ class ServiceExecutorSynchronousFixture : public unittest::Test { }; void scheduleBasicTask(ServiceExecutor* exec, bool expectSuccess) { - stdx::condition_variable cond; - stdx::mutex mutex; - auto task = [&cond, &mutex] { - stdx::unique_lock lk(mutex); - cond.notify_all(); + struct Completion { + stdx::condition_variable cond; + stdx::mutex mutex; + bool completed = false; + }; + auto completion = std::make_shared(); + auto task = [completion] { + stdx::unique_lock lk(completion->mutex); + completion->completed = true; + completion->cond.notify_all(); }; - stdx::unique_lock lk(mutex); + stdx::unique_lock lk(completion->mutex); auto status = exec->schedule( std::move(task), ServiceExecutor::kEmptyFlags, ServiceExecutorTaskName::kSSMStartSession); if (expectSuccess) { ASSERT_OK(status); - cond.wait(lk); + ASSERT_TRUE(completion->cond.wait_for( + lk, std::chrono::seconds(10), [&] { return completion->completed; })); } else { ASSERT_NOT_OK(status); } } -TEST_F(ServiceExecutorAdaptiveFixture, BasicTaskRuns) { +TEST_F(ServiceExecutorAdaptiveFixture, RunsEachIngressReactor) { ASSERT_OK(executor->start()); - auto guard = MakeGuard([this] { ASSERT_OK(executor->shutdown(kShutdownTime)); }); + for (const auto& reactor : reactors) { + auto completed = std::make_shared>(); + auto future = completed->get_future(); + reactor->schedule(Reactor::kPost, [completed] { completed->set_value(); }); + ASSERT_TRUE(future.wait_for(std::chrono::seconds(10)) == stdx::future_status::ready); + future.get(); + } + ASSERT_EQ(reactors.size(), static_cast(executor->threadsRunning())); + ASSERT_OK(executor->shutdown(kShutdownTime)); + ASSERT_EQ(0, executor->threadsRunning()); +} - scheduleBasicTask(executor.get(), true); +TEST_F(ServiceExecutorAdaptiveFixture, ShutdownBeforeStartupIsSafe) { + ASSERT_OK(executor->shutdown(kShutdownTime)); } -TEST_F(ServiceExecutorAdaptiveFixture, ScheduleFailsBeforeStartup) { - scheduleBasicTask(executor.get(), false); +TEST_F(ServiceExecutorAdaptiveFixture, ShutdownIsIdempotent) { + ASSERT_OK(executor->start()); + ASSERT_OK(executor->shutdown(kShutdownTime)); + ASSERT_OK(executor->shutdown(kShutdownTime)); +} + +TEST_F(ServiceExecutorAdaptiveFixture, RejectsEmptyReactorList) { + ServiceExecutorAdaptive empty(getGlobalServiceContext(), {}, + stdx::make_unique()); + ASSERT_EQ(ErrorCodes::InvalidOptions, empty.start().code()); + ASSERT_OK(empty.shutdown(kShutdownTime)); +} + +DEATH_TEST_F(ServiceExecutorAdaptiveFixture, SchedulingIsUnreachable, "MONGO_UNREACHABLE") { + ASSERT_OK(executor->start()); + executor->schedule([] {}, ServiceExecutor::kEmptyFlags, + ServiceExecutorTaskName::kSSMStartSession).ignore(); +} + +#ifdef ELOQ_MODULE_ENABLED +TEST(ServiceExecutorCoroutine, ModuleSchedulingAndLifecycle) { + constexpr int kThreadGroups = 2; + // Start only brpc's worker pool: no Data Substrate, storage engine, or data-store fixture. + // This matches the module-backed scheduler used by the EloqDoc server. + FLAGS_brpc_worker_as_ext_processor = true; + ASSERT_EQ(0, bthread_setconcurrency(kThreadGroups)); + bthread_t bootstrap; + ASSERT_EQ(0, bthread_start_background( + &bootstrap, nullptr, [](void*) -> void* { return nullptr; }, nullptr)); + ASSERT_EQ(0, bthread_join(bootstrap, nullptr)); + + auto service = ServiceContext::make(); + ServiceExecutorCoroutine executor(service.get(), kThreadGroups); + bool rejectedTaskRan = false; + auto rejectedTask = [&] { rejectedTaskRan = true; }; + const auto flags = ServiceExecutor::kEmptyFlags; + const auto taskName = ServiceExecutorTaskName::kSSMStartSession; + ASSERT_EQ(ErrorCodes::ShutdownInProgress, + executor.schedule(rejectedTask, flags, taskName).code()); + ASSERT_EQ(ErrorCodes::ShutdownInProgress, + executor.schedule(rejectedTask, flags, taskName, 1).code()); + + ASSERT_OK(executor.start()); + bool running = true; + auto stop = MakeGuard([&] { + if (running) + ASSERT_OK(executor.shutdown(kShutdownTime)); + }); + auto checkGroup = [](int16_t expectedGroup, auto submit) { + auto completed = std::make_shared>(); + auto future = completed->get_future(); + ServiceExecutor::Task task = [completed] { completed->set_value(LocalThread::ID()); }; + submit(task); + ASSERT_TRUE(future.wait_for(std::chrono::seconds(10)) == stdx::future_status::ready); + ASSERT_EQ(expectedGroup, future.get()); + }; + checkGroup(0, [&](const ServiceExecutor::Task& task) { + ASSERT_OK(executor.schedule(task, flags, taskName)); + }); + for (int16_t group = 0; group < kThreadGroups; ++group) { + checkGroup(group, [&](const ServiceExecutor::Task& task) { + ASSERT_OK(executor.schedule(task, flags, taskName, group)); + }); + checkGroup(group, [&](const ServiceExecutor::Task& task) { + executor.coroutineResumeFunctor(group, task)(); + }); + checkGroup(group, [&](const ServiceExecutor::Task& task) { + executor.coroutineLongResumeFunctor(group, task)(); + }); + checkGroup(group, [&](const ServiceExecutor::Task& task) { + executor.deferCallOnMainStack(group, ServiceExecutor::Task(task)); + }); + } + ASSERT_OK(executor.shutdown(kShutdownTime)); + running = false; + ASSERT_EQ(ErrorCodes::ShutdownInProgress, + executor.schedule(rejectedTask, flags, taskName).code()); + ASSERT_EQ(ErrorCodes::ShutdownInProgress, + executor.schedule(rejectedTask, flags, taskName, 1).code()); + ASSERT_FALSE(rejectedTaskRan); } +#endif TEST_F(ServiceExecutorSynchronousFixture, BasicTaskRuns) { ASSERT_OK(executor->start()); diff --git a/src/mongo/transport/service_state_machine.cpp b/src/mongo/transport/service_state_machine.cpp index e60ffa4185..d539bf6126 100644 --- a/src/mongo/transport/service_state_machine.cpp +++ b/src/mongo/transport/service_state_machine.cpp @@ -58,6 +58,7 @@ #include #include #include +#include namespace mongo { namespace { @@ -315,6 +316,7 @@ void ServiceStateMachine::reset(ServiceContext* svcContext, _coroLongResume = {}; _coroMigrateThreadGroup = {}; _resumeTask = {}; + _dbResponse = {}; _migrating.store(false, std::memory_order_relaxed); _threadGroupId.store(groupId, std::memory_order_relaxed); _owned.store(Ownership::kUnowned); @@ -467,7 +469,6 @@ void ServiceStateMachine::_processMessage(ThreadGuard guard) { // Pass sourced Message to handler to generate response. - DbResponse dbresponse; { // OperationContext opCtx(Client::getCurrent(), _serviceContext->nextOpId()); // _serviceContext->initOperationContext(&opCtx); @@ -477,11 +478,12 @@ void ServiceStateMachine::_processMessage(ThreadGuard guard) { // The handleRequest is implemented in a subclass for mongod/mongos and actually all the // database work for this request. - dbresponse = _sep->handleRequest(opCtx.get(), _inMessage); + _dbResponse = _sep->handleRequest(opCtx.get(), _inMessage); // Calling reset() to wait for Eloq-Tx be committed/aborted before set _coroStatus to Empty. opCtx.reset(nullptr); + invariant(_coroStatus == CoroStatus::OnGoing); _coroStatus = CoroStatus::Empty; _serviceExecutor->ongoingCoroutineCountUpdate( _threadGroupId.load(std::memory_order_relaxed), -1); @@ -491,6 +493,11 @@ void ServiceStateMachine::_processMessage(ThreadGuard guard) { // opCtx.reset(); // _serviceContext->destoryOperationContext(&opCtx); } +} + +void ServiceStateMachine::_processResponse(ThreadGuard guard) { + invariant(!_source); + auto dbresponse = std::move(_dbResponse); // Format our response, if we have one Message& toSink = dbresponse.response; @@ -510,11 +517,12 @@ void ServiceStateMachine::_processMessage(ThreadGuard guard) { networkCounter.hitLogicalOut(toSink.size()); if (_compressorId) { + auto& compressorMgr = MessageCompressorManager::forSession(_session()); auto swm = compressorMgr.compressMessage(toSink, &_compressorId.value()); uassertStatusOK(swm.getStatus()); toSink = swm.getValue(); } - _sinkMessage(std::move(guard), std::move(toSink)); + return _sinkMessage(std::move(guard), std::move(toSink)); } else { _state.store(State::Source); @@ -551,29 +559,16 @@ void ServiceStateMachine::_runNextInGuard(ThreadGuard guard) { case State::Process: { if (!serverGlobalParams.enableCoroutine) { _processMessage(std::move(guard)); + return _processResponse(ThreadGuard(this)); } else { if (_coroStatus == CoroStatus::Empty) { MONGO_LOG(1) << "coroutine begin"; _coroStatus = CoroStatus::OnGoing; _serviceExecutor->ongoingCoroutineCountUpdate( _threadGroupId.load(std::memory_order_relaxed), 1); - _resumeTask = [ssm = this] { - Client::setCurrent(std::move(ssm->_dbClient)); - if (ssm->_migrating.load(std::memory_order_relaxed)) { - ssm->_coroResume(); - ssm->_coroYield(); - } else { - ssm->_runResumeProcess(); - bool migrating = true; - ssm->_migrating.compare_exchange_strong( - migrating, false, std::memory_order_acq_rel); - if (migrating) { - MONGO_LOG(1) - << "Migrate logical session to " - << ssm->_threadGroupId.load(std::memory_order_relaxed); - } - } - }; + // Session ownership lasts until final cleanup. Do not access the SSM + // after this call: it may finish the request and release that ownership. + _resumeTask = [ssm = this] { ssm->_runResumeProcess(); }; _coroResume = _serviceExecutor->coroutineResumeFunctor( _threadGroupId.load(std::memory_order_relaxed), _resumeTask); @@ -588,44 +583,38 @@ void ServiceStateMachine::_runNextInGuard(ThreadGuard guard) { std::allocator_arg, prealloc, NoopAllocator(), - [wssm = weak_from_this(), &guard](boost::context::continuation&& sink) { - auto ssm = wssm.lock(); - if (!ssm) { - return std::move(sink); - } - ssm->_coroYield = [ssm = ssm.get(), &sink]() { + [ssm = this, &guard](boost::context::continuation&& sink) { + // Session ownership lasts until this coroutine returns and final + // cleanup runs on the executor stack. + ssm->_coroYield = [ssm, &sink]() { MONGO_LOG(3) << "call yield"; ssm->_dbClient = Client::releaseCurrent(); sink = sink.resume(); }; - ssm->_processMessage(std::move(guard)); + try { + ssm->_processMessage(std::move(guard)); + } catch (const DBException&) { + // Keep cleanup on the executor stack, including when a request + // throws after yielding and resuming on this coroutine stack. + ssm->_coroutineException = std::current_exception(); + if (ssm->_coroStatus == CoroStatus::OnGoing) { + ssm->_coroStatus = CoroStatus::Empty; + ssm->_serviceExecutor->ongoingCoroutineCountUpdate( + ssm->_threadGroupId.load(std::memory_order_relaxed), -1); + } + } catch (const std::exception& e) { + error() << "Uncaught std::exception: " << e.what() + << ", terminating"; + quickExit(EXIT_UNCAUGHT); + } return std::move(sink); }); - - bool migrating = true; - _migrating.compare_exchange_strong( - migrating, false, std::memory_order_acq_rel); - if (migrating) { - MONGO_LOG(1) << "Migrate logical session to " - << _threadGroupId.load(std::memory_order_relaxed); - } - - // _source = - // boost::context::callcc([this, &guard](boost::context::continuation&& - // sink) { - // _coroYield = [this, &sink]() { - // MONGO_LOG(1) << "call yield"; - // _dbClient = Client::releaseCurrent(); - // sink = sink.resume(); - // }; - // _processMessage(std::move(guard)); - // return std::move(sink); - // }); } else if (_coroStatus == CoroStatus::OnGoing) { MONGO_LOG(1) << "coroutine ongoing"; _source = _source.resume(); } + return _finishCoroutineStep(); } } break; case State::EndSession: @@ -653,12 +642,53 @@ void ServiceStateMachine::_runNextInGuard(ThreadGuard guard) { void ServiceStateMachine::_runResumeProcess() { MONGO_LOG(3) << "ServiceStateMachine::_resumeRun"; + Client::setCurrent(std::move(_dbClient)); + if (_migrating.load(std::memory_order_relaxed)) { + _coroResume(); + _coroYield(); + return; + } + if (_coroStatus == CoroStatus::OnGoing) { MONGO_LOG(3) << "coroutine ongoing"; _source = _source.resume(); + return _finishCoroutineStep(); } } +void ServiceStateMachine::_finishCoroutineStep() { + bool migrating = true; + _migrating.compare_exchange_strong(migrating, false, std::memory_order_acq_rel); + if (migrating) { + MONGO_LOG(1) << "Migrate logical session to " + << _threadGroupId.load(std::memory_order_relaxed); + } + + if (_coroStatus == CoroStatus::OnGoing) { + // The request yielded. Keep the session registered until it resumes and finishes. + return; + } + invariant(!_source); + + try { + if (_coroutineException) { + std::rethrow_exception(std::exchange(_coroutineException, nullptr)); + } + // All coroutine and resume bookkeeping is complete. Response delivery can invoke + // cleanup inline, so this must be the last operation that accesses the SSM. + return _processResponse(ThreadGuard(this)); + } catch (const DBException& e) { + log() << "DBException handling request, closing client connection: " << redact(e); + } catch (const std::exception& e) { + error() << "Uncaught std::exception: " << e.what() << ", terminating"; + quickExit(EXIT_UNCAUGHT); + } + + _state.store(State::EndSession); + // This can destroy the SSM. No access to its members is allowed after final cleanup. + return _cleanupSession(ThreadGuard(this)); +} + void ServiceStateMachine::start(Ownership ownershipModel) { MONGO_LOG(1) << "ServiceStateMachine::start"; _scheduleNextWithGuard(ThreadGuard(this), @@ -779,9 +809,12 @@ void ServiceStateMachine::_terminateAndLogIfError(Status status) { void ServiceStateMachine::_cleanupSession(ThreadGuard guard) { MONGO_LOG(1) << "ServiceStateMachine::_cleanupSession"; + invariant(_coroStatus == CoroStatus::Empty); + invariant(!_source); _state.store(State::Ended); _inMessage.reset(); + _dbResponse = {}; // By ignoring the return value of Client::releaseCurrent() we destroy the session. // _dbClient is now nullptr and _dbClientPtr is invalid and should never be accessed. diff --git a/src/mongo/transport/service_state_machine.h b/src/mongo/transport/service_state_machine.h index e68db8f705..a099a68ab3 100644 --- a/src/mongo/transport/service_state_machine.h +++ b/src/mongo/transport/service_state_machine.h @@ -32,6 +32,7 @@ #include #include #include +#include #include #include "boost/optional/optional.hpp" @@ -213,12 +214,21 @@ class ServiceStateMachine : public std::enable_shared_from_this _compressorId; Message _inMessage; + DbResponse _dbResponse; AtomicWord _owned{Ownership::kUnowned}; #if MONGO_CONFIG_DEBUG_BUILD @@ -286,6 +297,9 @@ class ServiceStateMachine : public std::enable_shared_from_thisgetClient()->coroutineFunctors(); + (*coro.resumeFuncPtr)(); + (*coro.yieldFuncPtr)(); + } if (_uassertInHandler) uassert(40469, "Synthetic uassert failure", false); @@ -100,6 +114,10 @@ class MockSEP : public ServiceEntryPoint { _uassertInHandler = true; } + void setYieldInHandler() { + _yieldInHandler = true; + } + bool ranHandler() { bool ret = _ranHandler; _ranHandler = false; @@ -108,6 +126,7 @@ class MockSEP : public ServiceEntryPoint { private: bool _uassertInHandler = false; + bool _yieldInHandler = false; bool _ranHandler = false; }; @@ -242,8 +261,29 @@ class MockServiceExecutor : public ServiceExecutor { _scheduleHook = std::move(hook); } + std::function coroutineResumeFunctor(uint16_t, const Task& task) override { + return [this, task] { _resumeTask = task; }; + } + + void ongoingCoroutineCountUpdate(uint16_t, int delta) override { + _ongoingCoroutines += delta; + ASSERT_GTE(_ongoingCoroutines, 0); + } + + int ongoingCoroutines() const { + return _ongoingCoroutines; + } + + void resumeCoroutine() { + auto task = std::move(_resumeTask); + ASSERT_TRUE(static_cast(task)); + task(); + } + private: ScheduleHook _scheduleHook; + Task _resumeTask; + int _ongoingCoroutines = 0; }; class SimpleEvent { @@ -304,6 +344,20 @@ class ServiceStateMachineFixture : public unittest::Test { void runPingTest(State first, State second); void checkPingOk(); + std::weak_ptr expectFinalCleanup() { + std::weak_ptr weakSSM = _ssm; + _ssm->setCleanupHook([this, weakSSM] { + ASSERT_FALSE(haveClient()); + ASSERT_EQ(_sexec->ongoingCoroutines(), 0); + // Only the session owner remains: neither the coroutine nor its resume callback + // may hold an extra reference that masks premature cleanup. + ASSERT_EQ(weakSSM.use_count(), 1); + _ssm.reset(); + ASSERT_TRUE(weakSSM.expired()); + }); + return weakSSM; + } + MockTL* _tl; MockSEP* _sep; MockServiceExecutor* _sexec; @@ -337,6 +391,7 @@ void ServiceStateMachineFixture::checkPingOk() { TEST_F(ServiceStateMachineFixture, TestOkaySimpleCommand) { runPingTest(State::Process, State::Source); checkPingOk(); + ASSERT_EQ(_sexec->ongoingCoroutines(), 0); } TEST_F(ServiceStateMachineFixture, TestThrowHandling) { @@ -346,6 +401,79 @@ TEST_F(ServiceStateMachineFixture, TestThrowHandling) { ASSERT(_tl->getLastSunk().empty()); ASSERT_TRUE(_tl->ranSource()); ASSERT_FALSE(_tl->ranSink()); + ASSERT_EQ(_sexec->ongoingCoroutines(), 0); +} + +TEST_F(ServiceStateMachineFixture, TestThrowAfterCoroutineResumeReleasesSession) { + _sep->setYieldInHandler(); + _sep->setUassertInHandler(); + auto weakSSM = expectFinalCleanup(); + + runPingTest(State::Process, State::Process); + ASSERT_EQ(_sexec->ongoingCoroutines(), 1); + _sexec->resumeCoroutine(); + + ASSERT_FALSE(haveClient()); + ASSERT_TRUE(weakSSM.expired()); + ASSERT_EQ(_sexec->ongoingCoroutines(), 0); + ASSERT_TRUE(_tl->getLastSunk().empty()); + ASSERT_FALSE(_tl->ranSink()); +} + +TEST_F(ServiceStateMachineFixture, TestThrowBeforeCoroutineYieldReleasesSession) { + _sep->setUassertInHandler(); + auto weakSSM = expectFinalCleanup(); + + _ssm->runNext(); + ASSERT_EQ(State::Process, _ssm->state()); + _ssm->runNext(); + + ASSERT_TRUE(weakSSM.expired()); + ASSERT_FALSE(_tl->ranSink()); +} + +TEST_F(ServiceStateMachineFixture, TestSinkErrorAfterCoroutineReturnReleasesSession) { + _tl->setNextFailure(MockTL::Sink); + auto weakSSM = expectFinalCleanup(); + + _ssm->runNext(); + ASSERT_EQ(State::Process, _ssm->state()); + _ssm->runNext(); + + ASSERT_TRUE(weakSSM.expired()); + ASSERT_TRUE(_tl->ranSink()); +} + +TEST_F(ServiceStateMachineFixture, TestSinkErrorAfterCoroutineResumeReleasesSession) { + _sep->setYieldInHandler(); + _tl->setNextFailure(MockTL::Sink); + auto weakSSM = expectFinalCleanup(); + + runPingTest(State::Process, State::Process); + ASSERT_EQ(_sexec->ongoingCoroutines(), 1); + _sexec->resumeCoroutine(); + + ASSERT_TRUE(weakSSM.expired()); + ASSERT_TRUE(_tl->ranSink()); +} + +TEST_F(ServiceStateMachineFixture, TerminateWhileCoroutineSuspendedDefersCleanup) { + _sep->setYieldInHandler(); + auto weakSSM = expectFinalCleanup(); + + runPingTest(State::Process, State::Process); + // The session owner alone keeps the suspended SSM alive; the coroutine does not own itself. + ASSERT_EQ(weakSSM.use_count(), 1); + _ssm->terminate(); + ASSERT_FALSE(weakSSM.expired()); + ASSERT_EQ(State::Process, _ssm->state()); + ASSERT_EQ(_sexec->ongoingCoroutines(), 1); + + // Termination closes the socket but must leave the suspended request alive to finish. + _sexec->resumeCoroutine(); + ASSERT_TRUE(weakSSM.expired()); + ASSERT_TRUE(_tl->ranSink()); + ASSERT_TRUE(_tl->getLastSunk().empty()); } TEST_F(ServiceStateMachineFixture, TestSourceError) { diff --git a/src/mongo/transport/transport_layer_asio_test.cpp b/src/mongo/transport/transport_layer_asio_test.cpp index b9ff5580dc..4e0d2cb46a 100644 --- a/src/mongo/transport/transport_layer_asio_test.cpp +++ b/src/mongo/transport/transport_layer_asio_test.cpp @@ -46,6 +46,14 @@ namespace { class ServiceEntryPointUtil : public ServiceEntryPoint { public: + Status start() override { + return Status::OK(); + } + + transport::ServiceExecutor* getServiceExecutor() override { + return nullptr; + } + void startSession(transport::SessionHandle session) override { stdx::unique_lock lk(_mutex); _sessions.push_back(std::move(session)); @@ -161,6 +169,14 @@ TEST(TransportLayerASIO, PortZeroConnect) { class TimeoutSEP : public ServiceEntryPoint { public: + Status start() override { + return Status::OK(); + } + + transport::ServiceExecutor* getServiceExecutor() override { + return nullptr; + } + void endAllSessions(transport::Session::TagMask tags) override { MONGO_UNREACHABLE; } diff --git a/src/mongo/unittest/death_test.cpp b/src/mongo/unittest/death_test.cpp index e9def2e6e7..807b124959 100644 --- a/src/mongo/unittest/death_test.cpp +++ b/src/mongo/unittest/death_test.cpp @@ -38,6 +38,10 @@ #include #endif +#ifdef __linux__ +#include +#endif + #if defined(__APPLE__) #include #endif @@ -120,6 +124,12 @@ void DeathTestImpl::_doTest() { // process calls std::abort() or std::terminate(). const struct rlimit kNoCoreDump { 0U, 0U }; checkSyscall(setrlimit(RLIMIT_CORE, &kNoCoreDump)); +#ifdef __linux__ + // Linux ignores RLIMIT_CORE for piped core_pattern collectors (including WSL's). + // Expected deaths must not launch a crash collector or dump this large test process. + // Only this child becomes nondumpable; unexpected parent/server crashes retain diagnostics. + checkSyscall(prctl(PR_SET_DUMPABLE, 0, 0, 0, 0)); +#endif try { auto test = _makeTest(); diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript index c9414e0c28..2712581e18 100644 --- a/src/mongo/util/SConscript +++ b/src/mongo/util/SConscript @@ -310,6 +310,7 @@ env.CppUnitTest( ], LIBDEPS=[ "background_job", + "$BUILD_DIR/mongo/db/service_context", ], ) diff --git a/src/mongo/util/clock_source.cpp b/src/mongo/util/clock_source.cpp index d562089c27..4f24a0b419 100644 --- a/src/mongo/util/clock_source.cpp +++ b/src/mongo/util/clock_source.cpp @@ -35,53 +35,6 @@ namespace mongo { stdx::cv_status ClockSource::waitForConditionUntil(stdx::condition_variable& cv, stdx::unique_lock& m, Date_t deadline) { - if (_tracksSystemClock) { - return cv.wait_until(m, deadline.toSystemTimePoint()); - } - - // The rest of this function only runs during testing, when the clock source is virtualized and - // does not track the system clock. - - if (deadline <= now()) { - return stdx::cv_status::timeout; - } - - struct AlarmInfo { - stdx::mutex controlMutex; - stdx::mutex* waitMutex; - stdx::condition_variable* waitCV; - stdx::cv_status cvWaitResult = stdx::cv_status::no_timeout; - }; - auto alarmInfo = std::make_shared(); - alarmInfo->waitCV = &cv; - alarmInfo->waitMutex = m.mutex(); - const auto waiterThreadId = stdx::this_thread::get_id(); - bool invokedAlarmInline = false; - invariant(setAlarm(deadline, [alarmInfo, waiterThreadId, &invokedAlarmInline] { - stdx::lock_guard controlLk(alarmInfo->controlMutex); - alarmInfo->cvWaitResult = stdx::cv_status::timeout; - if (!alarmInfo->waitMutex) { - return; - } - if (stdx::this_thread::get_id() == waiterThreadId) { - // In NetworkInterfaceMock, setAlarm may invoke its callback immediately if the deadline - // has expired, so we detect that case and avoid self-deadlock by returning early, here. - // It is safe to set invokedAlarmInline without synchronization in this case, because it - // is exactly the case where the same thread is writing and consulting the value. - invokedAlarmInline = true; - return; - } - stdx::lock_guard waitLk(*alarmInfo->waitMutex); - alarmInfo->waitCV->notify_all(); - })); - if (!invokedAlarmInline) { - cv.wait(m); - } - m.unlock(); - stdx::lock_guard controlLk(alarmInfo->controlMutex); - m.lock(); - alarmInfo->waitMutex = nullptr; - alarmInfo->waitCV = nullptr; - return alarmInfo->cvWaitResult; + return waitForConditionUntil(cv, m, deadline); } } // namespace mongo diff --git a/src/mongo/util/clock_source.h b/src/mongo/util/clock_source.h index e38ce7c932..6135ee3a0d 100644 --- a/src/mongo/util/clock_source.h +++ b/src/mongo/util/clock_source.h @@ -28,9 +28,11 @@ #pragma once +#include #include "mongo/stdx/condition_variable.h" #include "mongo/stdx/functional.h" #include "mongo/stdx/mutex.h" +#include "mongo/stdx/thread.h" #include "mongo/util/time_support.h" namespace mongo { @@ -81,6 +83,52 @@ class ClockSource { stdx::unique_lock& m, Date_t deadline); + // Also support coroutine-aware locks/condition variables without bypassing a + // virtual clock. The same alarm protocol must be used by both kinds of waiter. + template + stdx::cv_status waitForConditionUntil(ConditionVariable& cv, + stdx::unique_lock& m, + Date_t deadline) { + if (_tracksSystemClock) + return cv.wait_until(m, deadline.toSystemTimePoint()); + if (deadline <= now()) + return stdx::cv_status::timeout; + + struct AlarmInfo { + stdx::mutex controlMutex; + Mutex* waitMutex; + ConditionVariable* waitCV; + stdx::cv_status result = stdx::cv_status::no_timeout; + }; + auto alarm = std::make_shared(); + alarm->waitCV = &cv; + alarm->waitMutex = m.mutex(); + const auto waiterThreadId = stdx::this_thread::get_id(); + bool invokedInline = false; + invariant(setAlarm(deadline, [alarm, waiterThreadId, &invokedInline] { + stdx::lock_guard control(alarm->controlMutex); + alarm->result = stdx::cv_status::timeout; + if (!alarm->waitMutex) + return; + // A mock clock may invoke an already-expired alarm in setAlarm(). + // The caller already owns waitMutex, so it must not lock it again. + if (stdx::this_thread::get_id() == waiterThreadId) { + invokedInline = true; + return; + } + stdx::lock_guard waiting(*alarm->waitMutex); + alarm->waitCV->notify_all(); + })); + if (!invokedInline) + cv.wait(m); + m.unlock(); + stdx::lock_guard control(alarm->controlMutex); + m.lock(); + alarm->waitMutex = nullptr; + alarm->waitCV = nullptr; + return alarm->result; + } + /** * Like cv.wait_until(m, deadline, pred), but uses this ClockSource instead of * stdx::chrono::system_clock to measure the passage of time. diff --git a/src/mongo/util/options_parser/options_parser_test.cpp b/src/mongo/util/options_parser/options_parser_test.cpp index edbece614e..13d893c2da 100644 --- a/src/mongo/util/options_parser/options_parser_test.cpp +++ b/src/mongo/util/options_parser/options_parser_test.cpp @@ -835,7 +835,10 @@ TEST(Style, NoGuessing) { argv.push_back("--hel"); std::map env_map; - ASSERT_NOT_OK(parser.run(testOpts, argv, env_map, &environment)); + // Eloq allows unregistered options for the Data Substrate's flag parser. An unknown + // abbreviation may pass through, but must not be guessed as the registered --help flag. + ASSERT_OK(parser.run(testOpts, argv, env_map, &environment)); + ASSERT_FALSE(environment.count("help")); } TEST(Style, LongDisguises) { diff --git a/src/mongo/util/unordered_fast_key_table.h b/src/mongo/util/unordered_fast_key_table.h index c47ffadd05..dd7c53d342 100644 --- a/src/mongo/util/unordered_fast_key_table.h +++ b/src/mongo/util/unordered_fast_key_table.h @@ -76,7 +76,7 @@ class UnorderedFastKeyTable { Entry() = default; Entry(const Entry& other) - : _used(other._used), _everUsed(other.everUsed), _curHash(other._curHash) { + : _used(other._used), _everUsed(other._everUsed), _curHash(other._curHash) { if (other.isUsed()) { new (&_data) value_type(other.getData()); } diff --git a/tests/jstests/core/set_param1.js b/tests/jstests/core/set_param1.js index 06b1365cbe..0d4edaefde 100644 --- a/tests/jstests/core/set_param1.js +++ b/tests/jstests/core/set_param1.js @@ -124,7 +124,12 @@ assert.commandWorked( db.adminCommand({"setParameter": 1, logComponentVerbosity: old.logComponentVerbosity})); var isMongos = (db.isMaster().msg === 'isdbgrid'); -if (!isMongos) { +// The standalone EloqDoc build omits MongoDB replication commands and parameters. +// Keep all generic checks above. Probe the command set, not the parameters under test, +// so a missing oplog-fetcher parameter still fails on builds that include replication. +var supportsReplication = assert.commandWorked(db.adminCommand({listCommands: 1})) + .commands.hasOwnProperty('replSetGetStatus'); +if (!isMongos && supportsReplication) { // // oplogFetcherSteadyStateMaxFetcherRestarts //