diff --git a/.github/workflows/native-backend-ci.yml b/.github/workflows/native-backend-ci.yml index 8ed7976c..561d0c8d 100644 --- a/.github/workflows/native-backend-ci.yml +++ b/.github/workflows/native-backend-ci.yml @@ -24,33 +24,38 @@ jobs: - name: Install build tools run: | - python -m pip install -U pip + python -m pip install -q -U pip # numpy pinned <2.5: the R parity cache keys hash test inputs generated via # multivariate_normal (LAPACK SVD). numpy 2.5.x bundles an OpenBLAS whose SVD # kernels differ on some runner CPUs, changing inputs bit-for-bit and causing # cache misses. Re-evaluate at the next full live-R cache regeneration. - python -m pip install build scikit-build-core nanobind pytest ruff mypy "numpy<2.5" scipy - python -m pip install hypothesis pytest-benchmark pytest-xdist + python -m pip install -q build scikit-build-core nanobind pytest ruff mypy "numpy<2.5" scipy + python -m pip install -q hypothesis pytest-benchmark pytest-xdist - name: Install package editable - run: python -m pip install -e . + run: python -m pip install -q -e . - name: Run native import smoke test run: python -c "import nns._nnscore as c; print(c.lpm(2.0, 0.0, [-2.0, -1.0, 0.5, 3.0]))" - name: Run invariants without Rscript - run: python -m pytest -q tests/invariants + run: | + set +e + python -m pytest -q --tb=short tests/invariants > invariant-results.txt 2>&1 + status=$? + cat invariant-results.txt + exit $status - name: Run parity from committed R cache - run: NNS_R_CACHE_ONLY=1 python -m pytest -q tests/parity + run: NNS_R_CACHE_ONLY=1 python -m pytest -q --tb=short tests/parity - name: Run plotting color-fidelity tests - run: python -m pytest -q tests/plotting + run: python -m pytest -q --tb=short tests/plotting - name: Run vignette examples run: | if [ -f tests/docs/test_vignette_examples.py ]; then - python -m pytest -q tests/docs/test_vignette_examples.py + python -m pytest -q --tb=short tests/docs/test_vignette_examples.py else echo "No docs vignette test present; skipping." fi diff --git a/.github/workflows/repaired-r-parity.yml b/.github/workflows/repaired-r-parity.yml new file mode 100644 index 00000000..1f2541d5 --- /dev/null +++ b/.github/workflows/repaired-r-parity.yml @@ -0,0 +1,100 @@ +name: Repaired R parity fixtures + +on: + workflow_dispatch: + pull_request: + paths: + - "src/nns/**" + - "tests/parity/**" + - ".github/workflows/repaired-r-parity.yml" + +jobs: + repaired-r-fixtures: + runs-on: ubuntu-latest + env: + R_NNS_REPOSITORY: OVVO-Financial/NNS + REPAIRED_NNS_R_SHA: "21be6d92d8ad23f0848191b094aded0dd6df8f74" + FIXTURE_DIR: tests/parity/fixtures/repaired_r_13_1_21be6d92 + R_VALIDATION_LOG_DIR: artifacts/repaired-r-validation + PYTEST_LOG: pytest-21be6d92.txt + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: 'release' + + - uses: astral-sh/setup-uv@v5 + + - name: Install Python dependencies + run: uv sync --all-extras --dev + + - name: Install R dependencies + run: | + Rscript -e 'install.packages(c("devtools", "Rcpp", "jsonlite", "remotes", "digest", "desc"), repos = "https://cloud.r-project.org")' + + - name: Validate pinned R SHA variable + run: | + if ! printf '%s' "${REPAIRED_NNS_R_SHA}" | grep -Eq '^[0-9a-f]{40}$'; then + echo "REPAIRED_NNS_R_SHA must be a 40-character lowercase git SHA" >&2 + exit 1 + fi + + - name: Checkout repaired R NNS reference + run: | + git clone https://github.com/${R_NNS_REPOSITORY}.git ../NNS-r + cd ../NNS-r + git checkout "${REPAIRED_NNS_R_SHA}" + test "$(git rev-parse HEAD)" = "${REPAIRED_NNS_R_SHA}" + echo "R_NNS_COMMIT=${REPAIRED_NNS_R_SHA}" >> "$GITHUB_ENV" + + - name: Validate repaired R package + working-directory: ../NNS-r + run: | + mkdir -p "${GITHUB_WORKSPACE}/${R_VALIDATION_LOG_DIR}" + Rscript "${GITHUB_WORKSPACE}/tests/parity/check_repaired_r_docs.R" 2>&1 | tee "${GITHUB_WORKSPACE}/${R_VALIDATION_LOG_DIR}/00-doc-contract-before-document.log" + Rscript -e 'Rcpp::compileAttributes()' 2>&1 | tee "${GITHUB_WORKSPACE}/${R_VALIDATION_LOG_DIR}/01-compileAttributes.log" + Rscript -e 'devtools::document()' 2>&1 | tee "${GITHUB_WORKSPACE}/${R_VALIDATION_LOG_DIR}/02-document.log" + Rscript "${GITHUB_WORKSPACE}/tests/parity/check_repaired_r_docs.R" 2>&1 | tee "${GITHUB_WORKSPACE}/${R_VALIDATION_LOG_DIR}/02b-doc-contract-after-document.log" + Rscript -e 'devtools::load_all()' 2>&1 | tee "${GITHUB_WORKSPACE}/${R_VALIDATION_LOG_DIR}/03-load_all.log" + Rscript -e 'devtools::test()' 2>&1 | tee "${GITHUB_WORKSPACE}/${R_VALIDATION_LOG_DIR}/04-test.log" + Rscript -e 'devtools::check(error_on = "error", document = FALSE)' 2>&1 | tee "${GITHUB_WORKSPACE}/${R_VALIDATION_LOG_DIR}/05-check.log" + + - name: Validate repaired R stack candidate invariants + working-directory: ../NNS-r + run: | + mkdir -p "${GITHUB_WORKSPACE}/${R_VALIDATION_LOG_DIR}" + Rscript "${GITHUB_WORKSPACE}/tests/parity/check_repaired_r_stack_invariants.R" \ + "${GITHUB_WORKSPACE}/${R_VALIDATION_LOG_DIR}" \ + 2>&1 | tee "${GITHUB_WORKSPACE}/${R_VALIDATION_LOG_DIR}/06-stack-invariants.log" + + - name: Generate repaired R fixtures with reference backend + run: | + Rscript tests/parity/generate_repaired_r_fixtures.R \ + --r-repo ../NNS-r \ + --out "${FIXTURE_DIR}" \ + --commit "${R_NNS_COMMIT}" + + - name: Verify repaired R fixtures + run: uv run python tests/parity/verify_repaired_r_fixtures.py "${FIXTURE_DIR}" + + - name: Run Python parity tests against repaired fixtures + run: uv run pytest -q tests/parity -m parity 2>&1 | tee "${PYTEST_LOG}" + + - name: Upload R validation logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: repaired-r-validation-${{ env.R_NNS_COMMIT }} + path: ${{ env.R_VALIDATION_LOG_DIR }} + if-no-files-found: error + + - name: Upload repaired fixture artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: repaired-r-13-1-21be6d92-fixtures-${{ env.R_NNS_COMMIT }} + path: | + ${{ env.FIXTURE_DIR }} + ${{ env.PYTEST_LOG }} + if-no-files-found: warn diff --git a/artifacts/python_54c98418_failure_inventory.csv b/artifacts/python_54c98418_failure_inventory.csv new file mode 100644 index 00000000..355918bb --- /dev/null +++ b/artifacts/python_54c98418_failure_inventory.csv @@ -0,0 +1,56 @@ +test,test_case,function,field,previous_expectation,python_actual,r_54c98418_actual,fixture_case,classification,root_cause,implementation_change,test_fixture_change,resolution_commit +tests/parity/test_boost.py,test_nns_boost_numeric_matches_r[None],NNS.boost,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,boost_numeric,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_boost.py,test_nns_boost_numeric_matches_r[1],NNS.boost,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,boost_numeric,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[50-2-linear-None-None-None-False-off],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,numeric_l2_default,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[50-3-nonlinear-1-1-point_est1-False-off],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,numeric_l2_default,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_boost.py,test_nns_boost_numeric_matches_r[2],NNS.boost,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,boost_numeric,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[200-3-mixed-2-2-None-False-mean],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,numeric_l2_default,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[200-5-linear-max-None-None-False-median],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,numeric_order_max,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_boost.py,test_nns_boost_ivs_test_none_matches_r,NNS.boost,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,boost_class_pred_int,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[50-2-nonlinear-1-1-point_est4-True-off],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,numeric_l2_default,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_confidence_interval_matches_r[2-0.8-None-None-None],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,numeric_l2_default,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_confidence_interval_matches_r[3-0.95-None-2-None],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,numeric_l2_default,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_confidence_interval_matches_r[2-0.95-1-1-point_est2],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,numeric_l2_default,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_confidence_interval_matches_r[3-0.8-2-2-point_est3],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,numeric_l2_default,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_classification_matches_r[2-classes0-point_est0-1-1],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,multiclass,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_classification_matches_r[2-classes0-point_est0-1-2],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,multiclass,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_classification_matches_r[3-classes1-point_est1-2-1],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,multiclass,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_classification_matches_r[3-classes1-point_est1-2-2],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,multiclass,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_class_confidence_interval_matches_r[2-classes0-point_est0-1-1],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,multiclass,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_class_confidence_interval_matches_r[2-classes0-point_est0-1-2],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,multiclass,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_class_confidence_interval_matches_r[3-classes1-point_est1-2-1],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,multiclass,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_class_confidence_interval_matches_r[3-classes1-point_est1-2-2],multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,multiclass,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_factor_levels_return_numeric_codes,multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,multiclass,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_m_reg_factor_levels_class_confidence_interval_matches_r,multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,multiclass,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_multivariate_regression.py,test_nns_reg_matrix_classification_dispatches_to_m_reg,multivariate regression,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,multiclass,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_boost.py,test_nns_boost_ts_test_deterministic_matches_r[3],NNS.boost,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,boost_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_boost.py,test_nns_boost_ts_test_deterministic_matches_r[5],NNS.boost,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,boost_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_boost.py,test_nns_boost_ts_test_deterministic_matches_r[8],NNS.boost,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,boost_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_r13_smoke.py,test_r_nns_13_seeded_stack_smoke_sample,NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method1_regression,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_regression.py,test_nns_reg_factor_predictor_matches_r_full_rank_dummy_path,NNS.reg,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,reg_default,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_boost.py,test_nns_boost_numeric_pred_int_matches_r[1-0.95],NNS.boost,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,boost_numeric,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_boost.py,test_nns_boost_numeric_pred_int_matches_r[2-0.8],NNS.boost,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,boost_numeric,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method2-5],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method12_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method3-10],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method12_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_numeric_matches_r[True-method0],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method1_regression,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method4-10],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method12_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_numeric_matches_r[True-method2],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method12_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_var_like_ts_test_matches_r,NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method12_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_boost.py,test_nns_boost_binary_class_pred_int_matches_r[1],NNS.boost,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,boost_class_pred_int,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_pred_int_matches_r[method0],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_pred_int,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_numeric_matches_r[False-method0],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method1_regression,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_boost.py,test_nns_boost_binary_class_pred_int_matches_r[2],NNS.boost,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,boost_class_pred_int,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_pred_int_matches_r[method2],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method12_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_numeric_matches_r[False-method2],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method12_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_binary_class_matches_r[method0],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_classification,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_mixed_factor_predictor_method12_matches_r,NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method1_regression,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method0-5],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method12_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_binary_class_pred_int_matches_r[method0],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_classification,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method1-10],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method12_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_multiclass_matches_r[method2],NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_method12_ts,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_factor_like_class_pred_int_matches_r,NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_classification,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_stack.py,test_nns_stack_factor_like_class_matches_r,NNS.stack,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,stack_classification,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_var.py,test_var_interpolate_and_extrapolate_matches_r[trailing_na-3],NNS.VAR,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,var_cor_missing,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_var.py,test_var_multivariate_stack_stage_matches_r[tau1-1-cor],NNS.VAR,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,var_cor_tau1,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_var.py,test_public_nns_var_cor_handles_missing_values_like_r,NNS.VAR,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,var_cor_missing,F,unresolved pending executable 54c98418 R fixture comparison,,, +tests/parity/test_var.py,test_public_nns_var_cor_matches_r[scalar_tau-1],NNS.VAR,public parity assertion,cached parity expectation from pre-54c98418 fixture set,see pytest-54c98418.txt failure traceback,unavailable until repaired R workflow artifacts are imported,var_cor_tau1,F,unresolved pending executable 54c98418 R fixture comparison,,, diff --git a/artifacts/python_54c98418_failure_inventory.md b/artifacts/python_54c98418_failure_inventory.md new file mode 100644 index 00000000..3c00d572 --- /dev/null +++ b/artifacts/python_54c98418_failure_inventory.md @@ -0,0 +1,64 @@ +# Python 54c98418 repaired parity failure inventory + +All rows remain category F because executable R 54c98418 fixture artifacts have not been imported in this environment. Do not update expectations from this inventory alone. + +- Total failures: 55 +- R commit: 54c98418c2a11499ebb1c456570d2b66c37eb817 + +| # | test | function | fixture case | classification | action | +|---:|---|---|---|---|---| +| 1 | `tests/parity/test_boost.py::test_nns_boost_numeric_matches_r[None]` | NNS.boost | `boost_numeric` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 2 | `tests/parity/test_boost.py::test_nns_boost_numeric_matches_r[1]` | NNS.boost | `boost_numeric` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 3 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_matches_r[50-2-linear-None-None-None-False-off]` | multivariate regression | `numeric_l2_default` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 4 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_matches_r[50-3-nonlinear-1-1-point_est1-False-off]` | multivariate regression | `numeric_l2_default` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 5 | `tests/parity/test_boost.py::test_nns_boost_numeric_matches_r[2]` | NNS.boost | `boost_numeric` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 6 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_matches_r[200-3-mixed-2-2-None-False-mean]` | multivariate regression | `numeric_l2_default` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 7 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_matches_r[200-5-linear-max-None-None-False-median]` | multivariate regression | `numeric_order_max` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 8 | `tests/parity/test_boost.py::test_nns_boost_ivs_test_none_matches_r` | NNS.boost | `boost_class_pred_int` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 9 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_matches_r[50-2-nonlinear-1-1-point_est4-True-off]` | multivariate regression | `numeric_l2_default` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 10 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_confidence_interval_matches_r[2-0.8-None-None-None]` | multivariate regression | `numeric_l2_default` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 11 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_confidence_interval_matches_r[3-0.95-None-2-None]` | multivariate regression | `numeric_l2_default` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 12 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_confidence_interval_matches_r[2-0.95-1-1-point_est2]` | multivariate regression | `numeric_l2_default` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 13 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_confidence_interval_matches_r[3-0.8-2-2-point_est3]` | multivariate regression | `numeric_l2_default` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 14 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_classification_matches_r[2-classes0-point_est0-1-1]` | multivariate regression | `multiclass` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 15 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_classification_matches_r[2-classes0-point_est0-1-2]` | multivariate regression | `multiclass` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 16 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_classification_matches_r[3-classes1-point_est1-2-1]` | multivariate regression | `multiclass` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 17 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_classification_matches_r[3-classes1-point_est1-2-2]` | multivariate regression | `multiclass` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 18 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_class_confidence_interval_matches_r[2-classes0-point_est0-1-1]` | multivariate regression | `multiclass` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 19 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_class_confidence_interval_matches_r[2-classes0-point_est0-1-2]` | multivariate regression | `multiclass` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 20 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_class_confidence_interval_matches_r[3-classes1-point_est1-2-1]` | multivariate regression | `multiclass` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 21 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_class_confidence_interval_matches_r[3-classes1-point_est1-2-2]` | multivariate regression | `multiclass` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 22 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_factor_levels_return_numeric_codes` | multivariate regression | `multiclass` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 23 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_factor_levels_class_confidence_interval_matches_r` | multivariate regression | `multiclass` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 24 | `tests/parity/test_multivariate_regression.py::test_nns_reg_matrix_classification_dispatches_to_m_reg` | multivariate regression | `multiclass` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 25 | `tests/parity/test_boost.py::test_nns_boost_ts_test_deterministic_matches_r[3]` | NNS.boost | `boost_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 26 | `tests/parity/test_boost.py::test_nns_boost_ts_test_deterministic_matches_r[5]` | NNS.boost | `boost_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 27 | `tests/parity/test_boost.py::test_nns_boost_ts_test_deterministic_matches_r[8]` | NNS.boost | `boost_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 28 | `tests/parity/test_r13_smoke.py::test_r_nns_13_seeded_stack_smoke_sample` | NNS.stack | `stack_method1_regression` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 29 | `tests/parity/test_regression.py::test_nns_reg_factor_predictor_matches_r_full_rank_dummy_path` | NNS.reg | `reg_default` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 30 | `tests/parity/test_boost.py::test_nns_boost_numeric_pred_int_matches_r[1-0.95]` | NNS.boost | `boost_numeric` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 31 | `tests/parity/test_boost.py::test_nns_boost_numeric_pred_int_matches_r[2-0.8]` | NNS.boost | `boost_numeric` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 32 | `tests/parity/test_stack.py::test_nns_stack_ts_test_matches_r[method2-5]` | NNS.stack | `stack_method12_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 33 | `tests/parity/test_stack.py::test_nns_stack_ts_test_matches_r[method3-10]` | NNS.stack | `stack_method12_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 34 | `tests/parity/test_stack.py::test_nns_stack_numeric_matches_r[True-method0]` | NNS.stack | `stack_method1_regression` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 35 | `tests/parity/test_stack.py::test_nns_stack_ts_test_matches_r[method4-10]` | NNS.stack | `stack_method12_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 36 | `tests/parity/test_stack.py::test_nns_stack_numeric_matches_r[True-method2]` | NNS.stack | `stack_method12_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 37 | `tests/parity/test_stack.py::test_nns_stack_var_like_ts_test_matches_r` | NNS.stack | `stack_method12_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 38 | `tests/parity/test_boost.py::test_nns_boost_binary_class_pred_int_matches_r[1]` | NNS.boost | `boost_class_pred_int` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 39 | `tests/parity/test_stack.py::test_nns_stack_pred_int_matches_r[method0]` | NNS.stack | `stack_pred_int` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 40 | `tests/parity/test_stack.py::test_nns_stack_numeric_matches_r[False-method0]` | NNS.stack | `stack_method1_regression` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 41 | `tests/parity/test_boost.py::test_nns_boost_binary_class_pred_int_matches_r[2]` | NNS.boost | `boost_class_pred_int` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 42 | `tests/parity/test_stack.py::test_nns_stack_pred_int_matches_r[method2]` | NNS.stack | `stack_method12_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 43 | `tests/parity/test_stack.py::test_nns_stack_numeric_matches_r[False-method2]` | NNS.stack | `stack_method12_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 44 | `tests/parity/test_stack.py::test_nns_stack_binary_class_matches_r[method0]` | NNS.stack | `stack_classification` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 45 | `tests/parity/test_stack.py::test_nns_stack_mixed_factor_predictor_method12_matches_r` | NNS.stack | `stack_method1_regression` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 46 | `tests/parity/test_stack.py::test_nns_stack_ts_test_matches_r[method0-5]` | NNS.stack | `stack_method12_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 47 | `tests/parity/test_stack.py::test_nns_stack_binary_class_pred_int_matches_r[method0]` | NNS.stack | `stack_classification` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 48 | `tests/parity/test_stack.py::test_nns_stack_ts_test_matches_r[method1-10]` | NNS.stack | `stack_method12_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 49 | `tests/parity/test_stack.py::test_nns_stack_multiclass_matches_r[method2]` | NNS.stack | `stack_method12_ts` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 50 | `tests/parity/test_stack.py::test_nns_stack_factor_like_class_pred_int_matches_r` | NNS.stack | `stack_classification` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 51 | `tests/parity/test_stack.py::test_nns_stack_factor_like_class_matches_r` | NNS.stack | `stack_classification` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 52 | `tests/parity/test_var.py::test_var_interpolate_and_extrapolate_matches_r[trailing_na-3]` | NNS.VAR | `var_cor_missing` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 53 | `tests/parity/test_var.py::test_var_multivariate_stack_stage_matches_r[tau1-1-cor]` | NNS.VAR | `var_cor_tau1` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 54 | `tests/parity/test_var.py::test_public_nns_var_cor_handles_missing_values_like_r` | NNS.VAR | `var_cor_missing` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | +| 55 | `tests/parity/test_var.py::test_public_nns_var_cor_matches_r[scalar_tau-1]` | NNS.VAR | `var_cor_tau1` | F | Import 54c98418 R artifacts, compare actuals, then reclassify. | diff --git a/artifacts/python_repaired_failure_inventory.csv b/artifacts/python_repaired_failure_inventory.csv new file mode 100644 index 00000000..e3da63a8 --- /dev/null +++ b/artifacts/python_repaired_failure_inventory.csv @@ -0,0 +1,56 @@ +test path,test name,affected public function,affected returned field,old expected value,new actual value,downstream of stack,downstream of multivariate regression,executable repaired R output available,classification,action taken +tests/parity/test_boost.py,test_nns_boost_numeric_matches_r[None],nns_boost,boost estimates/intervals,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_boost.py,test_nns_boost_numeric_matches_r[1],nns_boost,boost estimates/intervals,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[50-2-linear-None-None-None-False-off],nns_m_reg,numeric predictions/RPM/R2,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_boost.py,test_nns_boost_numeric_matches_r[2],nns_boost,boost estimates/intervals,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[50-3-nonlinear-1-1-point_est1-False-off],nns_m_reg,numeric predictions/RPM/R2,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_boost.py,test_nns_boost_ivs_test_none_matches_r,nns_boost,boost estimates/intervals,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[200-3-mixed-2-2-None-False-mean],nns_m_reg,numeric predictions/RPM/R2,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[200-5-linear-max-None-None-False-median],nns_m_reg,numeric predictions/RPM/R2,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[50-2-nonlinear-1-1-point_est4-True-off],nns_m_reg,numeric predictions/RPM/R2,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_confidence_interval_matches_r[2-0.8-None-None-None],nns_m_reg,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_confidence_interval_matches_r[3-0.95-None-2-None],nns_m_reg,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_confidence_interval_matches_r[2-0.95-1-1-point_est2],nns_m_reg,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_confidence_interval_matches_r[3-0.8-2-2-point_est3],nns_m_reg,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_classification_matches_r[2-classes0-point_est0-1-1],nns_m_reg,classification predictions/RPM/Fitted.xy,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_classification_matches_r[2-classes0-point_est0-1-2],nns_m_reg,classification predictions/RPM/Fitted.xy,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_classification_matches_r[3-classes1-point_est1-2-1],nns_m_reg,classification predictions/RPM/Fitted.xy,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_classification_matches_r[3-classes1-point_est1-2-2],nns_m_reg,classification predictions/RPM/Fitted.xy,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_class_confidence_interval_matches_r[2-classes0-point_est0-1-1],nns_m_reg,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_class_confidence_interval_matches_r[2-classes0-point_est0-1-2],nns_m_reg,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_class_confidence_interval_matches_r[3-classes1-point_est1-2-1],nns_m_reg,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_class_confidence_interval_matches_r[3-classes1-point_est1-2-2],nns_m_reg,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_factor_levels_return_numeric_codes,nns_m_reg,factor encoded predictions/RPM,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_m_reg_factor_levels_class_confidence_interval_matches_r,nns_m_reg,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_multivariate_regression.py,test_nns_reg_matrix_classification_dispatches_to_m_reg,nns_reg,classification predictions/RPM/Fitted.xy,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. +tests/parity/test_boost.py,test_nns_boost_ts_test_deterministic_matches_r[3],nns_boost,ts_test fold predictions/selected parameters,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_boost.py,test_nns_boost_ts_test_deterministic_matches_r[5],nns_boost,ts_test fold predictions/selected parameters,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_boost.py,test_nns_boost_ts_test_deterministic_matches_r[8],nns_boost,ts_test fold predictions/selected parameters,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_r13_smoke.py,test_r_nns_13_seeded_stack_smoke_sample,nns_stack,numeric predictions/RPM/R2,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_boost.py,test_nns_boost_numeric_pred_int_matches_r[1-0.95],nns_boost,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_boost.py,test_nns_boost_numeric_pred_int_matches_r[2-0.8],nns_boost,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_regression.py,test_nns_reg_factor_predictor_matches_r_full_rank_dummy_path,nns_reg,factor encoded predictions/RPM,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,False,True,no,F,regression factor path mismatch after shared mreg/interval changes; no fixture update made pending executable repaired R. +tests/parity/test_boost.py,test_nns_boost_binary_class_pred_int_matches_r[1],nns_boost,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method2-5],nns_stack,ts_test fold predictions/selected parameters,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method3-10],nns_stack,ts_test fold predictions/selected parameters,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_numeric_matches_r[True-method0],nns_stack,numeric predictions/RPM/R2,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_boost.py,test_nns_boost_binary_class_pred_int_matches_r[2],nns_boost,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method4-10],nns_stack,ts_test fold predictions/selected parameters,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_numeric_matches_r[True-method2],nns_stack,numeric predictions/RPM/R2,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_var_like_ts_test_matches_r,nns_stack,ts_test fold predictions/selected parameters,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_numeric_matches_r[False-method0],nns_stack,numeric predictions/RPM/R2,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_pred_int_matches_r[method0],nns_stack,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_pred_int_matches_r[method2],nns_stack,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_numeric_matches_r[False-method2],nns_stack,numeric predictions/RPM/R2,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_mixed_factor_predictor_method12_matches_r,nns_stack,factor encoded predictions/RPM,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_binary_class_matches_r[method0],nns_stack,classification predictions/RPM/Fitted.xy,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method0-5],nns_stack,ts_test fold predictions/selected parameters,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method1-10],nns_stack,ts_test fold predictions/selected parameters,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_binary_class_pred_int_matches_r[method0],nns_stack,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_multiclass_matches_r[method2],nns_stack,classification predictions/RPM/Fitted.xy,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_factor_like_class_pred_int_matches_r,nns_stack,prediction interval and/or prediction fields,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_stack.py,test_nns_stack_factor_like_class_matches_r,nns_stack,classification predictions/RPM/Fitted.xy,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. +tests/parity/test_var.py,test_var_interpolate_and_extrapolate_matches_r[trailing_na-3],nns_var,VAR multivariate/ensemble output,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,VAR downstream mismatch because VAR consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_var.py,test_var_multivariate_stack_stage_matches_r[tau1-1-cor],nns_var,VAR multivariate/ensemble output,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,VAR downstream mismatch because VAR consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_var.py,test_public_nns_var_cor_handles_missing_values_like_r,nns_var,VAR multivariate/ensemble output,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,VAR downstream mismatch because VAR consumes stack/regression internals; no fixture update made pending executable repaired R. +tests/parity/test_var.py,test_public_nns_var_cor_matches_r[scalar_tau-1],nns_var,VAR multivariate/ensemble output,See pytest-full.txt failure block for cached/R expected value; executable repaired R not available locally.,See pytest-full.txt failure block for Python actual value from current branch.,True,True,no,F,VAR downstream mismatch because VAR consumes stack/regression internals; no fixture update made pending executable repaired R. diff --git a/artifacts/python_repaired_failure_inventory.md b/artifacts/python_repaired_failure_inventory.md new file mode 100644 index 00000000..75dfb49d --- /dev/null +++ b/artifacts/python_repaired_failure_inventory.md @@ -0,0 +1,66 @@ +# Python repaired failure inventory + +Captured command: `uv run pytest -q > pytest-full.txt 2>&1` +Total failures captured: 55 + +Classification key: A=Python bug; B=intended repaired behavior confirmed by executable R; C=obsolete R fixture; D=invalid implementation-detail test; E=unrelated regression; F=unresolved because repaired R has not yet been executed. + +All rows are currently classified F because this environment has no R runtime and no pinned executable repaired R fixture output. No expected values were updated. + +| # | Test | Public function | Field | Stack downstream | MReg downstream | Class | Action | +|---:|---|---|---|---|---|---|---| +| 1 | `tests/parity/test_boost.py::test_nns_boost_numeric_matches_r[None]` | `nns_boost` | boost estimates/intervals | True | True | F | boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 2 | `tests/parity/test_boost.py::test_nns_boost_numeric_matches_r[1]` | `nns_boost` | boost estimates/intervals | True | True | F | boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 3 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_matches_r[50-2-linear-None-None-None-False-off]` | `nns_m_reg` | numeric predictions/RPM/R2 | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 4 | `tests/parity/test_boost.py::test_nns_boost_numeric_matches_r[2]` | `nns_boost` | boost estimates/intervals | True | True | F | boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 5 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_matches_r[50-3-nonlinear-1-1-point_est1-False-off]` | `nns_m_reg` | numeric predictions/RPM/R2 | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 6 | `tests/parity/test_boost.py::test_nns_boost_ivs_test_none_matches_r` | `nns_boost` | boost estimates/intervals | True | True | F | boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 7 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_matches_r[200-3-mixed-2-2-None-False-mean]` | `nns_m_reg` | numeric predictions/RPM/R2 | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 8 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_matches_r[200-5-linear-max-None-None-False-median]` | `nns_m_reg` | numeric predictions/RPM/R2 | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 9 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_matches_r[50-2-nonlinear-1-1-point_est4-True-off]` | `nns_m_reg` | numeric predictions/RPM/R2 | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 10 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_confidence_interval_matches_r[2-0.8-None-None-None]` | `nns_m_reg` | prediction interval and/or prediction fields | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 11 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_confidence_interval_matches_r[3-0.95-None-2-None]` | `nns_m_reg` | prediction interval and/or prediction fields | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 12 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_confidence_interval_matches_r[2-0.95-1-1-point_est2]` | `nns_m_reg` | prediction interval and/or prediction fields | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 13 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_confidence_interval_matches_r[3-0.8-2-2-point_est3]` | `nns_m_reg` | prediction interval and/or prediction fields | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 14 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_classification_matches_r[2-classes0-point_est0-1-1]` | `nns_m_reg` | classification predictions/RPM/Fitted.xy | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 15 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_classification_matches_r[2-classes0-point_est0-1-2]` | `nns_m_reg` | classification predictions/RPM/Fitted.xy | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 16 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_classification_matches_r[3-classes1-point_est1-2-1]` | `nns_m_reg` | classification predictions/RPM/Fitted.xy | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 17 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_classification_matches_r[3-classes1-point_est1-2-2]` | `nns_m_reg` | classification predictions/RPM/Fitted.xy | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 18 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_class_confidence_interval_matches_r[2-classes0-point_est0-1-1]` | `nns_m_reg` | prediction interval and/or prediction fields | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 19 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_class_confidence_interval_matches_r[2-classes0-point_est0-1-2]` | `nns_m_reg` | prediction interval and/or prediction fields | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 20 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_class_confidence_interval_matches_r[3-classes1-point_est1-2-1]` | `nns_m_reg` | prediction interval and/or prediction fields | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 21 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_class_confidence_interval_matches_r[3-classes1-point_est1-2-2]` | `nns_m_reg` | prediction interval and/or prediction fields | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 22 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_factor_levels_return_numeric_codes` | `nns_m_reg` | factor encoded predictions/RPM | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 23 | `tests/parity/test_multivariate_regression.py::test_nns_m_reg_factor_levels_class_confidence_interval_matches_r` | `nns_m_reg` | prediction interval and/or prediction fields | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 24 | `tests/parity/test_multivariate_regression.py::test_nns_reg_matrix_classification_dispatches_to_m_reg` | `nns_reg` | classification predictions/RPM/Fitted.xy | False | True | F | direct multivariate regression mismatch after repaired interval/RPM preparation changes; no fixture update made pending executable repaired R. | +| 25 | `tests/parity/test_boost.py::test_nns_boost_ts_test_deterministic_matches_r[3]` | `nns_boost` | ts_test fold predictions/selected parameters | True | True | F | boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 26 | `tests/parity/test_boost.py::test_nns_boost_ts_test_deterministic_matches_r[5]` | `nns_boost` | ts_test fold predictions/selected parameters | True | True | F | boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 27 | `tests/parity/test_boost.py::test_nns_boost_ts_test_deterministic_matches_r[8]` | `nns_boost` | ts_test fold predictions/selected parameters | True | True | F | boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 28 | `tests/parity/test_r13_smoke.py::test_r_nns_13_seeded_stack_smoke_sample` | `nns_stack` | numeric predictions/RPM/R2 | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 29 | `tests/parity/test_boost.py::test_nns_boost_numeric_pred_int_matches_r[1-0.95]` | `nns_boost` | prediction interval and/or prediction fields | True | True | F | boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 30 | `tests/parity/test_boost.py::test_nns_boost_numeric_pred_int_matches_r[2-0.8]` | `nns_boost` | prediction interval and/or prediction fields | True | True | F | boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 31 | `tests/parity/test_regression.py::test_nns_reg_factor_predictor_matches_r_full_rank_dummy_path` | `nns_reg` | factor encoded predictions/RPM | False | True | F | regression factor path mismatch after shared mreg/interval changes; no fixture update made pending executable repaired R. | +| 32 | `tests/parity/test_boost.py::test_nns_boost_binary_class_pred_int_matches_r[1]` | `nns_boost` | prediction interval and/or prediction fields | True | True | F | boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 33 | `tests/parity/test_stack.py::test_nns_stack_ts_test_matches_r[method2-5]` | `nns_stack` | ts_test fold predictions/selected parameters | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 34 | `tests/parity/test_stack.py::test_nns_stack_ts_test_matches_r[method3-10]` | `nns_stack` | ts_test fold predictions/selected parameters | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 35 | `tests/parity/test_stack.py::test_nns_stack_numeric_matches_r[True-method0]` | `nns_stack` | numeric predictions/RPM/R2 | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 36 | `tests/parity/test_boost.py::test_nns_boost_binary_class_pred_int_matches_r[2]` | `nns_boost` | prediction interval and/or prediction fields | True | True | F | boost downstream mismatch because boost consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 37 | `tests/parity/test_stack.py::test_nns_stack_ts_test_matches_r[method4-10]` | `nns_stack` | ts_test fold predictions/selected parameters | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 38 | `tests/parity/test_stack.py::test_nns_stack_numeric_matches_r[True-method2]` | `nns_stack` | numeric predictions/RPM/R2 | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 39 | `tests/parity/test_stack.py::test_nns_stack_var_like_ts_test_matches_r` | `nns_stack` | ts_test fold predictions/selected parameters | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 40 | `tests/parity/test_stack.py::test_nns_stack_numeric_matches_r[False-method0]` | `nns_stack` | numeric predictions/RPM/R2 | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 41 | `tests/parity/test_stack.py::test_nns_stack_pred_int_matches_r[method0]` | `nns_stack` | prediction interval and/or prediction fields | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 42 | `tests/parity/test_stack.py::test_nns_stack_pred_int_matches_r[method2]` | `nns_stack` | prediction interval and/or prediction fields | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 43 | `tests/parity/test_stack.py::test_nns_stack_numeric_matches_r[False-method2]` | `nns_stack` | numeric predictions/RPM/R2 | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 44 | `tests/parity/test_stack.py::test_nns_stack_mixed_factor_predictor_method12_matches_r` | `nns_stack` | factor encoded predictions/RPM | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 45 | `tests/parity/test_stack.py::test_nns_stack_binary_class_matches_r[method0]` | `nns_stack` | classification predictions/RPM/Fitted.xy | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 46 | `tests/parity/test_stack.py::test_nns_stack_ts_test_matches_r[method0-5]` | `nns_stack` | ts_test fold predictions/selected parameters | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 47 | `tests/parity/test_stack.py::test_nns_stack_ts_test_matches_r[method1-10]` | `nns_stack` | ts_test fold predictions/selected parameters | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 48 | `tests/parity/test_stack.py::test_nns_stack_binary_class_pred_int_matches_r[method0]` | `nns_stack` | prediction interval and/or prediction fields | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 49 | `tests/parity/test_stack.py::test_nns_stack_multiclass_matches_r[method2]` | `nns_stack` | classification predictions/RPM/Fitted.xy | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 50 | `tests/parity/test_stack.py::test_nns_stack_factor_like_class_pred_int_matches_r` | `nns_stack` | prediction interval and/or prediction fields | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 51 | `tests/parity/test_stack.py::test_nns_stack_factor_like_class_matches_r` | `nns_stack` | classification predictions/RPM/Fitted.xy | True | True | F | stack output mismatch after repaired ts_test/Method 1/X-star/shared mreg changes; no fixture update made pending executable repaired R. | +| 52 | `tests/parity/test_var.py::test_var_interpolate_and_extrapolate_matches_r[trailing_na-3]` | `nns_var` | VAR multivariate/ensemble output | True | True | F | VAR downstream mismatch because VAR consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 53 | `tests/parity/test_var.py::test_var_multivariate_stack_stage_matches_r[tau1-1-cor]` | `nns_var` | VAR multivariate/ensemble output | True | True | F | VAR downstream mismatch because VAR consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 54 | `tests/parity/test_var.py::test_public_nns_var_cor_handles_missing_values_like_r` | `nns_var` | VAR multivariate/ensemble output | True | True | F | VAR downstream mismatch because VAR consumes stack/regression internals; no fixture update made pending executable repaired R. | +| 55 | `tests/parity/test_var.py::test_public_nns_var_cor_matches_r[scalar_tau-1]` | `nns_var` | VAR multivariate/ensemble output | True | True | F | VAR downstream mismatch because VAR consumes stack/regression internals; no fixture update made pending executable repaired R. | diff --git a/artifacts/repaired_failure_fixture_map.csv b/artifacts/repaired_failure_fixture_map.csv new file mode 100644 index 00000000..3118f890 --- /dev/null +++ b/artifacts/repaired_failure_fixture_map.csv @@ -0,0 +1,56 @@ +python_test,python_test_case,failure_family,r_fixture_file,r_fixture_case,intermediate_fields_available,coverage_status +tests/parity/test_boost.py,test_nns_boost_numeric_matches_r[None],NNS.boost,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,boost_numeric,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_boost.py,test_nns_boost_numeric_matches_r[1],NNS.boost,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,boost_numeric,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[50-2-linear-None-None-None-False-off],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,numeric_l2_default,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[50-3-nonlinear-1-1-point_est1-False-off],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,numeric_l2_default,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_boost.py,test_nns_boost_numeric_matches_r[2],NNS.boost,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,boost_numeric,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[200-3-mixed-2-2-None-False-mean],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,numeric_l2_default,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[200-5-linear-max-None-None-False-median],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,numeric_order_max,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_boost.py,test_nns_boost_ivs_test_none_matches_r,NNS.boost,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,boost_class_pred_int,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_matches_r[50-2-nonlinear-1-1-point_est4-True-off],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,numeric_l2_default,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_confidence_interval_matches_r[2-0.8-None-None-None],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,numeric_l2_default,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_confidence_interval_matches_r[3-0.95-None-2-None],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,numeric_l2_default,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_confidence_interval_matches_r[2-0.95-1-1-point_est2],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,numeric_l2_default,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_confidence_interval_matches_r[3-0.8-2-2-point_est3],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,numeric_l2_default,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_classification_matches_r[2-classes0-point_est0-1-1],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,multiclass,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_classification_matches_r[2-classes0-point_est0-1-2],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,multiclass,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_classification_matches_r[3-classes1-point_est1-2-1],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,multiclass,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_classification_matches_r[3-classes1-point_est1-2-2],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,multiclass,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_class_confidence_interval_matches_r[2-classes0-point_est0-1-1],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,multiclass,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_class_confidence_interval_matches_r[2-classes0-point_est0-1-2],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,multiclass,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_class_confidence_interval_matches_r[3-classes1-point_est1-2-1],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,multiclass,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_class_confidence_interval_matches_r[3-classes1-point_est1-2-2],multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,multiclass,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_factor_levels_return_numeric_codes,multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,multiclass,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_m_reg_factor_levels_class_confidence_interval_matches_r,multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,multiclass,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_multivariate_regression.py,test_nns_reg_matrix_classification_dispatches_to_m_reg,multivariate regression,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,multiclass,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_boost.py,test_nns_boost_ts_test_deterministic_matches_r[3],NNS.boost,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,boost_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_boost.py,test_nns_boost_ts_test_deterministic_matches_r[5],NNS.boost,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,boost_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_boost.py,test_nns_boost_ts_test_deterministic_matches_r[8],NNS.boost,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,boost_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_r13_smoke.py,test_r_nns_13_seeded_stack_smoke_sample,NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method1_regression,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_regression.py,test_nns_reg_factor_predictor_matches_r_full_rank_dummy_path,NNS.reg,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,reg_default,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_boost.py,test_nns_boost_numeric_pred_int_matches_r[1-0.95],NNS.boost,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,boost_numeric,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_boost.py,test_nns_boost_numeric_pred_int_matches_r[2-0.8],NNS.boost,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,boost_numeric,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method2-5],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method12_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method3-10],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method12_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_numeric_matches_r[True-method0],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method1_regression,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method4-10],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method12_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_numeric_matches_r[True-method2],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method12_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_var_like_ts_test_matches_r,NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method12_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_boost.py,test_nns_boost_binary_class_pred_int_matches_r[1],NNS.boost,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,boost_class_pred_int,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_pred_int_matches_r[method0],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_pred_int,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_numeric_matches_r[False-method0],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method1_regression,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_boost.py,test_nns_boost_binary_class_pred_int_matches_r[2],NNS.boost,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,boost_class_pred_int,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_pred_int_matches_r[method2],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method12_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_numeric_matches_r[False-method2],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method12_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_binary_class_matches_r[method0],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_classification,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_mixed_factor_predictor_method12_matches_r,NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method1_regression,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method0-5],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method12_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_binary_class_pred_int_matches_r[method0],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_classification,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_ts_test_matches_r[method1-10],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method12_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_multiclass_matches_r[method2],NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_method12_ts,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_factor_like_class_pred_int_matches_r,NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_classification,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_stack.py,test_nns_stack_factor_like_class_matches_r,NNS.stack,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,stack_classification,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_var.py,test_var_interpolate_and_extrapolate_matches_r[trailing_na-3],NNS.VAR,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,var_cor_missing,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_var.py,test_var_multivariate_stack_stage_matches_r[tau1-1-cor],NNS.VAR,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,var_cor_tau1,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_var.py,test_public_nns_var_cor_handles_missing_values_like_r,NNS.VAR,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,var_cor_missing,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture +tests/parity/test_var.py,test_public_nns_var_cor_matches_r[scalar_tau-1],NNS.VAR,tests/parity/fixtures/repaired_r_13_1_54c98418/fixtures.json,var_cor_tau1,"inputs,args,public output,input/output checksums; generator covers part/reg/mreg/stack/boost/var families and stores full R result objects",covered_by_planned_54c98418_repaired_r_fixture diff --git a/scripts/import_repaired_r_fixtures.py b/scripts/import_repaired_r_fixtures.py new file mode 100644 index 00000000..c6cf23ac --- /dev/null +++ b/scripts/import_repaired_r_fixtures.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +from pathlib import Path + +EXPECTED_REPOSITORY = "OVVO-Financial/NNS" +EXPECTED_SCHEMA = "repaired_r_13_1_21be6d92" +EXPECTED_REFERENCE_OPTIONS = { + "NNS.native.stack": False, + "NNS.native.mreg": False, + "NNS.native.univariate": False, +} +REQUIRED_FAMILIES = {"part", "reg", "mreg", "stack", "boost", "var"} + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def load_metadata(source: Path) -> dict[str, object]: + metadata_path = source / "metadata.json" + if not metadata_path.exists(): + raise SystemExit(f"missing fixture metadata: {metadata_path}") + return json.loads(metadata_path.read_text()) + + +def verify_metadata(metadata: dict[str, object], expected_commit: str) -> None: + if metadata.get("r_repository") != EXPECTED_REPOSITORY: + raise SystemExit(f"unexpected R repository: {metadata.get('r_repository')!r}") + if metadata.get("r_commit_sha") != expected_commit: + raise SystemExit( + f"unexpected R commit: {metadata.get('r_commit_sha')!r}; expected {expected_commit}" + ) + if metadata.get("fixture_schema_version") != EXPECTED_SCHEMA: + raise SystemExit(f"unexpected fixture schema: {metadata.get('fixture_schema_version')!r}") + options = metadata.get("native_reference_options") + if options != EXPECTED_REFERENCE_OPTIONS: + raise SystemExit(f"unexpected native/reference options: {options!r}") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Import verified repaired R fixture artifacts.") + parser.add_argument("artifact_dir", nargs="?", type=Path) + parser.add_argument("--artifact", dest="artifact", type=Path) + parser.add_argument("--expected-commit", dest="expected_commit") + parser.add_argument("--expected-r-sha", dest="expected_commit") + parser.add_argument( + "--dest", + type=Path, + default=Path("tests/parity/fixtures/repaired_r_13_1_21be6d92"), + ) + args = parser.parse_args() + artifact_dir = args.artifact or args.artifact_dir + if artifact_dir is None: + raise SystemExit("provide an artifact directory as a positional argument or --artifact") + if args.expected_commit is None: + raise SystemExit("--expected-r-sha is required") + valid_commit = len(args.expected_commit) == 40 and all( + c in "0123456789abcdef" for c in args.expected_commit + ) + if not valid_commit: + raise SystemExit("--expected-r-sha must be a 40-character lowercase git SHA") + metadata = load_metadata(artifact_dir) + verify_metadata(metadata, args.expected_commit) + fixtures_path = artifact_dir / "fixtures.json" + if not fixtures_path.exists(): + raise SystemExit(f"missing fixtures file: {fixtures_path}") + fixtures = json.loads(fixtures_path.read_text()) + cases = fixtures.get("cases", []) + names = [case.get("name") for case in cases] + if len(names) != len(set(names)): + raise SystemExit("fixture case names must be unique") + families = {str(case.get("kind")) for case in cases} + missing_families = REQUIRED_FAMILIES - families + if missing_families: + raise SystemExit( + "missing required fixture families: " + ", ".join(sorted(missing_families)) + ) + fixture_hashes = {path.name: sha256_file(path) for path in sorted(artifact_dir.glob("*.json"))} + args.dest.mkdir(parents=True, exist_ok=True) + for source in artifact_dir.iterdir(): + if source.is_file(): + shutil.copy2(source, args.dest / source.name) + (args.dest / "manifest.json").write_text( + json.dumps( + { + "schema_version": 1, + "r_repository": EXPECTED_REPOSITORY, + "r_commit": args.expected_commit, + "nns_version": metadata.get("nns_version"), + "reference_backend": {"stack": False, "mreg": False, "univariate": False}, + "fixture_hashes": fixture_hashes, + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + print(f"Imported repaired R fixtures into {args.dest}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/nns/__init__.py b/src/nns/__init__.py index e841ee7b..4d019bcf 100644 --- a/src/nns/__init__.py +++ b/src/nns/__init__.py @@ -91,3 +91,15 @@ def __getattr__(name: str) -> Any: value = getattr(import_module(module_name), attr_name) globals()[name] = value return value + + +# Install the repaired Method 1 implementation at package import time so both +# ``from nns import nns_stack`` and ``from nns.stack import nns_stack`` share +# the same pooled-OOF semantics without changing the public stack signature. +from importlib import import_module as _import_module + +_stack_module = _import_module("nns.stack") +_stack_runtime = _import_module("nns._stack_method1_runtime") +_stack_module._evaluate_method1 = _stack_runtime.evaluate_method1 + +del _import_module, _stack_module, _stack_runtime diff --git a/src/nns/_stack_method1.py b/src/nns/_stack_method1.py new file mode 100644 index 00000000..00d69f9f --- /dev/null +++ b/src/nns/_stack_method1.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import math +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Literal, TypeAlias + +import numpy as np +from numpy.typing import NDArray + +CandidateId: TypeAlias = int | Literal["all"] +Objective: TypeAlias = Literal["min", "max"] +CandidateEvaluator: TypeAlias = Callable[ + [CandidateId, NDArray[np.float64], NDArray[np.float64]], tuple[float, float] +] + + +@dataclass(frozen=True) +class Method1FoldPredictions: + """Predictions for one CV fold keyed by conceptual Method 1 candidate.""" + + validation_idx: NDArray[np.int64] + predictions: Mapping[CandidateId, NDArray[np.float64]] + + +@dataclass(frozen=True) +class Method1Selection: + """Complete pooled-OOF Method 1 candidate selection result.""" + + selected: CandidateId + selected_score: float + selected_threshold: float + pooled_predictions: Mapping[CandidateId, NDArray[np.float64]] + count_vectors: Mapping[CandidateId, NDArray[np.int64]] + scores: Mapping[CandidateId, float] + thresholds: Mapping[CandidateId, float] + eligible: tuple[CandidateId, ...] + excluded: tuple[CandidateId, ...] + stopping_k: int | None + + +def select_method1_candidate( + *, + n_obs: int, + actual: NDArray[np.float64], + folds: Sequence[Method1FoldPredictions], + local_candidates: Sequence[int], + evaluator: CandidateEvaluator, + objective: Objective, +) -> Method1Selection: + """Select Method 1 using complete pooled OOF predictions. + + The repaired R semantics are deliberately candidate-global: + + * each conceptual local ``k`` is accumulated across every fold; + * candidate 1 defines the required OOF coverage pattern; + * partial/empty candidates are excluded before scoring; + * diminishing-returns stopping is applied only to complete pooled scores; + * ``ALL`` is always scored separately and is never subject to local stopping. + """ + + if n_obs < 1: + raise ValueError("n_obs must be positive.") + actual_values = np.asarray(actual, dtype=np.float64).reshape(-1) + if actual_values.size != n_obs: + raise ValueError("actual must contain n_obs values.") + if objective not in {"min", "max"}: + raise ValueError("objective must be 'min' or 'max'.") + + local_ids = tuple(dict.fromkeys(int(k) for k in local_candidates if int(k) >= 1)) + if not local_ids or local_ids[0] != 1: + raise ValueError("local_candidates must begin with candidate 1.") + candidate_ids: tuple[CandidateId, ...] = (*local_ids, "all") + + sums = {candidate: np.zeros(n_obs, dtype=np.float64) for candidate in candidate_ids} + counts = {candidate: np.zeros(n_obs, dtype=np.int64) for candidate in candidate_ids} + + for fold in folds: + validation_idx = np.asarray(fold.validation_idx, dtype=np.int64).reshape(-1) + if np.any(validation_idx < 0) or np.any(validation_idx >= n_obs): + raise ValueError("fold validation indices are outside [0, n_obs).") + if np.unique(validation_idx).size != validation_idx.size: + raise ValueError("fold validation indices must be unique.") + + for candidate, raw_prediction in fold.predictions.items(): + if candidate not in sums: + continue + prediction = np.asarray(raw_prediction, dtype=np.float64).reshape(-1) + if prediction.size != validation_idx.size: + raise ValueError( + f"candidate {candidate!r} prediction length does not match validation indices." + ) + finite = np.isfinite(prediction) + if not np.any(finite): + continue + rows = validation_idx[finite] + sums[candidate][rows] += prediction[finite] + counts[candidate][rows] += 1 + + reference_count = counts[1] + if not np.any(reference_count > 0): + raise ValueError("candidate 1 has no OOF coverage.") + + pooled: dict[CandidateId, NDArray[np.float64]] = {} + scores: dict[CandidateId, float] = {} + thresholds: dict[CandidateId, float] = {} + complete: dict[CandidateId, bool] = {} + + for candidate in candidate_ids: + count = counts[candidate] + raw = np.full(n_obs, np.nan, dtype=np.float64) + covered = count > 0 + raw[covered] = sums[candidate][covered] / count[covered] + pooled[candidate] = raw + + same_coverage = np.array_equal(count, reference_count) + valid = covered & np.isfinite(raw) & np.isfinite(actual_values) + if not same_coverage or not np.any(valid): + complete[candidate] = False + continue + + score, threshold = evaluator(candidate, raw[valid], actual_values[valid]) + score_value = float(score) + threshold_value = float(threshold) + if not math.isfinite(score_value): + complete[candidate] = False + continue + complete[candidate] = True + scores[candidate] = score_value + thresholds[candidate] = threshold_value + + evaluated_local: list[int] = [] + stopping_k: int | None = None + for candidate in local_ids: + if not complete.get(candidate, False): + break + evaluated_local.append(candidate) + if len(evaluated_local) < 4: + continue + recent = [scores[k] for k in evaluated_local[-3:]] + stop = ( + recent[2] >= recent[1] and recent[2] >= recent[0] + if objective == "min" + else recent[2] <= recent[1] and recent[2] <= recent[0] + ) + if stop: + stopping_k = candidate + break + + eligible: list[CandidateId] = list(evaluated_local) + if complete.get("all", False): + eligible.append("all") + if not eligible: + raise ValueError("no Method 1 candidate has complete finite OOF coverage.") + + if objective == "min": + best_score = min(scores[candidate] for candidate in eligible) + else: + best_score = max(scores[candidate] for candidate in eligible) + selected = next(candidate for candidate in eligible if scores[candidate] == best_score) + + excluded = tuple(candidate for candidate in candidate_ids if candidate not in eligible) + return Method1Selection( + selected=selected, + selected_score=float(scores[selected]), + selected_threshold=float(thresholds[selected]), + pooled_predictions=pooled, + count_vectors=counts, + scores=scores, + thresholds=thresholds, + eligible=tuple(eligible), + excluded=excluded, + stopping_k=stopping_k, + ) diff --git a/src/nns/_stack_method1_runtime.py b/src/nns/_stack_method1_runtime.py new file mode 100644 index 00000000..862617ac --- /dev/null +++ b/src/nns/_stack_method1_runtime.py @@ -0,0 +1,261 @@ +from __future__ import annotations + +import math +from typing import Any, cast + +import numpy as np +from numpy.typing import NDArray + +from nns._stack_method1 import CandidateId, Method1FoldPredictions, select_method1_candidate +from nns.distance import nns_distance_path_single_bulk + + +def evaluate_method1( + x_train: NDArray[np.float64], + y_train: NDArray[np.float64], + x_test: NDArray[np.float64], + **kwargs: Any, +) -> Any: + """Repaired Method 1 implementation installed into :mod:`nns.stack`. + + Candidate selection is based on complete pooled OOF predictions. Local + candidates are bounded by ``floor(sqrt(n))`` and ``ALL`` is evaluated as a + separate single-k limit condition. Genuine univariate designs use the + univariate production estimator directly and do not invent a k path. + """ + + from nns import stack as s + + methods = kwargs["methods"] + objective = kwargs["objective"] + objective_fn = kwargs["objective_fn"] + cv_size = kwargs["cv_size"] + folds = kwargs["folds"] + order = kwargs["order"] + stack_enabled = kwargs["stack"] + dim_red_method = kwargs["dim_red_method"] + dist = kwargs["dist"] + method2_state = kwargs["method2_state"] + ts_test = kwargs["ts_test"] + pred_int = kwargs["pred_int"] + type_value = kwargs["type_value"] + mixed_factor = kwargs["mixed_factor"] + raw_columns = kwargs["raw_columns"] + status = kwargs.get("status", False) + ncores = kwargs.get("ncores", 1) + + if 1 not in methods: + obj = math.inf if objective == "min" else -math.inf + return s._MethodState(np.full(x_test.shape[0], np.nan), obj, math.nan) + + n_rows = x_train.shape[0] + local_candidates = tuple(range(1, max(1, math.floor(math.sqrt(n_rows))) + 1)) + fold_predictions: list[Method1FoldPredictions] = [] + + for fold in range(1, folds + 1): + train_idx, valid_idx = s._cv_split(n_rows, fold, cv_size, ts_test) + fold_x_train = x_train[train_idx] + fold_y_train = y_train[train_idx] + fold_x_valid = x_train[valid_idx] + fold_y_valid = y_train[valid_idx] + + if stack_enabled and methods == (1, 2) and method2_state.train_star is not None: + fold_train_star, fold_valid_star = s._fold_xstar( + fold_x_train, + fold_y_train, + fold_x_valid, + fold_y_valid, + mixed_factor=mixed_factor, + raw_columns=raw_columns, + objective=objective, + objective_fn=objective_fn, + order=order, + dim_red_method=dim_red_method, + dist=dist, + type_value=type_value, + ) + fold_x_train = np.column_stack((fold_train_star, fold_train_star)) + fold_x_valid = np.column_stack((fold_valid_star, fold_valid_star)) + elif method2_state.relevant_vars is not None and method2_state.relevant_vars.size: + fold_x_train = fold_x_train[:, method2_state.relevant_vars] + fold_x_valid = fold_x_valid[:, method2_state.relevant_vars] + + if fold_x_train.shape[1] == 1: + direct = s.nns_reg( + fold_x_train[:, 0], + fold_y_train, + point_est=fold_x_valid[:, 0], + order=None, + dist=dist, + point_only=True, + type=type_value, + ncores=ncores, + ) + prediction = s._as_prediction(direct["Point.est"], valid_idx.size) + fold_predictions.append( + Method1FoldPredictions(valid_idx, {1: prediction, "all": prediction.copy()}) + ) + continue + + model = s._mreg_prepare_model( + fold_x_train, + fold_y_train, + order=order, + noise_reduction="mode_class" if type_value == "class" else "off", + is_class=type_value == "class", + ) + local_counts = [max(1, min(k, model.rpm.shape[0])) for k in local_candidates] + local_path = s._mreg_predict_path( + model, + fold_x_valid, + k_values=local_counts, + is_class=type_value == "class", + ) + predictions: dict[CandidateId, NDArray[np.float64]] = { + k: np.asarray(local_path[count], dtype=np.float64).copy() + for k, count in zip(local_candidates, local_counts, strict=True) + } + class_arg = "class" if type_value == "class" else None + predictions["all"] = np.asarray( + nns_distance_path_single_bulk( + model.rpm, + fold_x_valid, + model.rpm.shape[0], + class_arg, + ), + dtype=np.float64, + ) + fold_predictions.append(Method1FoldPredictions(valid_idx, predictions)) + if status: + print(f"Method 1 fold {fold}/{folds} complete") + + if x_train.shape[1] == 1 and not ( + stack_enabled and methods == (1, 2) and method2_state.train_star is not None + ): + selected: CandidateId = 1 + pooled_score = math.nan + pooled_threshold = math.nan + # The selector is still used for coverage validation; ALL is an alias + # only inside this compatibility layer and cannot change the winner. + def univariate_eval( + candidate: CandidateId, + predicted: NDArray[np.float64], + actual: NDArray[np.float64], + ) -> tuple[float, float]: + threshold = ( + s._classification_threshold(predicted, actual, tie="first") + if type_value == "class" + else math.nan + ) + evaluated = ( + s._class_threshold_round(predicted, threshold, y_train) + if type_value == "class" + else predicted + ) + return float(objective_fn(evaluated, actual)), threshold + + selection = select_method1_candidate( + n_obs=n_rows, + actual=y_train, + folds=fold_predictions, + local_candidates=[1], + evaluator=univariate_eval, + objective=objective, + ) + pooled_score = selection.scores[1] + pooled_threshold = selection.thresholds[1] + else: + def evaluator( + candidate: CandidateId, + predicted: NDArray[np.float64], + actual: NDArray[np.float64], + ) -> tuple[float, float]: + if type_value != "class": + return float(objective_fn(predicted, actual)), math.nan + threshold = s._classification_threshold( + predicted, + actual, + tie="first" if candidate == 1 else "median", + ) + evaluated = s._class_threshold_round(predicted, threshold, y_train) + return float(objective_fn(evaluated, actual)), threshold + + selection = select_method1_candidate( + n_obs=n_rows, + actual=y_train, + folds=fold_predictions, + local_candidates=local_candidates, + evaluator=evaluator, + objective=objective, + ) + selected = selection.selected + pooled_score = selection.selected_score + pooled_threshold = selection.selected_threshold + + if stack_enabled and methods == (1, 2) and method2_state.train_star is not None: + if method2_state.test_star is None: + raise RuntimeError("stacked Method 1 requires Method 2 test projections.") + full_x_train = np.column_stack((method2_state.train_star, method2_state.train_star)) + full_x_test = np.column_stack((method2_state.test_star, method2_state.test_star)) + elif method2_state.relevant_vars is not None and method2_state.relevant_vars.size: + full_x_train = x_train[:, method2_state.relevant_vars] + full_x_test = x_test[:, method2_state.relevant_vars] + else: + full_x_train = x_train + full_x_test = x_test + + univariate = full_x_train.shape[1] == 1 + final_n_best: int | str | None = None if univariate else selected + final_x_train: NDArray[np.float64] = ( + full_x_train[:, 0] if univariate else full_x_train + ) + final_x_test: NDArray[np.float64] = full_x_test[:, 0] if univariate else full_x_test + final_fit = s.nns_reg( + final_x_train, + y_train, + point_est=final_x_test, + n_best=final_n_best, + order=None if univariate else order, + dist=dist, + point_only=False, + confidence_interval=pred_int, + type=type_value, + ncores=ncores, + ) + fitted = cast(dict[str, NDArray[np.float64]], final_fit["Fitted.xy"]) + prediction = s._as_prediction(final_fit["Point.est"], x_test.shape[0]) + fitted_yhat = fitted["y.hat"] + final_threshold = pooled_threshold + if type_value == "class": + if not np.isfinite(final_threshold): + final_threshold = s._classification_threshold(fitted_yhat, y_train) + fitted_yhat = s._class_threshold_round(fitted_yhat, final_threshold, y_train) + prediction = s._class_threshold_round(prediction, final_threshold, y_train) + final_obj = objective_fn(fitted_yhat, fitted["y"]) + final_pred_int = cast(dict[str, NDArray[np.float64]] | None, final_fit["pred.int"]) + final_pred_int = s._prediction_interval_or_point_estimate(final_pred_int, prediction) + + if selected == "all" and not univariate: + full_model = s._mreg_prepare_model( + full_x_train, + y_train, + order=order, + noise_reduction="mode_class" if type_value == "class" else "off", + is_class=type_value == "class", + ) + public_parameter = float(full_model.rpm.shape[0]) + else: + public_parameter = float(1 if univariate else selected) + + if status: + print( + f"Method 1 selected {selected!r} from pooled OOF score " + f"{pooled_score:.6g}" + ) + return s._MethodState( + prediction=prediction, + objective=float(final_obj), + parameter=public_parameter, + pred_int=final_pred_int, + class_threshold=final_threshold if type_value == "class" else None, + ) diff --git a/src/nns/multivariate_regression.py b/src/nns/multivariate_regression.py index 65c5bf5b..444e19a5 100644 --- a/src/nns/multivariate_regression.py +++ b/src/nns/multivariate_regression.py @@ -1,6 +1,8 @@ from __future__ import annotations import math +from collections.abc import Sequence +from dataclasses import dataclass from typing import Any, Literal, NotRequired, TypedDict, cast import numpy as np @@ -17,6 +19,109 @@ NBest = int | Literal["all"] | None + +@dataclass(frozen=True) +class MRegModel: + """Prepared multivariate NNS regression model shared by public APIs and stack.""" + + rpm_x: NDArray[np.float64] + rpm_y: NDArray[np.float64] + boundaries: tuple[NDArray[np.float64], ...] + ids: NDArray[np.str_] + minimums: NDArray[np.float64] + maximums: NDArray[np.float64] + feature_names: tuple[str, ...] + rpm: NDArray[np.float64] + + +def _mreg_prepare_model( + x: NDArray[np.float64], + y: NDArray[np.float64], + *, + order: Order, + noise_reduction: NoiseReduction, + is_class: bool, +) -> MRegModel: + """Prepare the invariant multivariate RPM/partition state once. + + This stripped path intentionally avoids default-k selection, fitted values, + external predictions, R2, residuals, confidence intervals, plots, and public + dictionaries so stack Method 1 can score all candidate k values without + rebuilding the model for every candidate. + """ + x_values = np.asarray(x, dtype=np.float64) + y_values = np.asarray(y, dtype=np.float64).reshape(-1) + if x_values.ndim != 2: + raise ValueError("x must be a 2D numeric matrix.") + if x_values.shape[0] == 0 or x_values.shape[1] == 0: + raise ValueError("x must be non-empty.") + if y_values.size != x_values.shape[0]: + raise ValueError("x and y must have the same row count.") + if not np.all(np.isfinite(x_values)) or not np.all(np.isfinite(y_values)): + raise ValueError("x and y must contain only finite values.") + + noise = _validate_noise(noise_reduction) + type_value = "class" if is_class else None + reg_points_matrix = _regression_points_matrix( + x_values, + y_values, + order, + noise, + False, + type_value, + ) + if order is None or isinstance(order, int): + reg_points_matrix = _unique_rows_preserve_order(reg_points_matrix) + boundaries = tuple( + np.sort(reg_points_matrix[:, col][np.isfinite(reg_points_matrix[:, col])]).astype( + np.float64, copy=True + ) + for col in range(x_values.shape[1]) + ) + components = _find_interval_matrix_from_boundaries(x_values, boundaries) + ids = _join_ids(components) + rpm = _rpm_from_ids( + x_values, + y_values, + ids, + noise, + order_is_numeric=order is None or isinstance(order, int), + ) + if is_class: + rpm = rpm.copy() + rpm[:, -1] = _round_clamp_classes(rpm[:, -1], y_values) + return MRegModel( + rpm_x=np.ascontiguousarray(rpm[:, :-1], dtype=np.float64), + rpm_y=np.ascontiguousarray(rpm[:, -1], dtype=np.float64), + boundaries=boundaries, + ids=ids, + minimums=np.min(x_values, axis=0), + maximums=np.max(x_values, axis=0), + feature_names=tuple(f"V{idx + 1}" for idx in range(x_values.shape[1])), + rpm=np.ascontiguousarray(rpm, dtype=np.float64), + ) + + +def _mreg_predict_path( + model: MRegModel, + x_test: NDArray[np.float64], + *, + k_values: Sequence[int], + is_class: bool, +) -> dict[int, NDArray[np.float64]]: + """Pure-Python repaired all-k reference path using one prepared RPM.""" + tests = np.asarray(x_test, dtype=np.float64) + if tests.ndim == 1: + tests = tests.reshape(1, -1) + if tests.ndim != 2 or tests.shape[1] != model.rpm_x.shape[1]: + raise ValueError("x_test column count must match prepared model.") + class_arg = "class" if is_class else None + out: dict[int, NDArray[np.float64]] = {} + for k in sorted({max(1, min(int(k), model.rpm.shape[0])) for k in k_values}): + out[k] = nns_distance_path_single_bulk(model.rpm, tests, k, class_arg) + return out + + MRegFitted = dict[str, NDArray[np.float64] | NDArray[np.str_]] """Fit table keyed ``V1..Vn`` per regressor plus ``y``, ``y.hat``, ``NNS.ID``, ``residuals``, and (with ``confidence_interval``) ``conf.int.pos``/``conf.int.neg``. @@ -77,7 +182,7 @@ def nns_m_reg( dist=dist != "L2", return_values=return_values is not False, plot_regions=plot_regions, - ncores=ncores is not None, + ncores=ncores is not None and int(ncores) > 1, ) type_value = _normalize_type(type) x_values, y_values = _validate_inputs( @@ -90,27 +195,29 @@ def nns_m_reg( point_values, point_is_matrix = _validate_point_est(point_est, x_values.shape[1]) noise = _validate_noise(noise_reduction) - reg_points_matrix = _regression_points_matrix( + model = _mreg_prepare_model( x_values, y_values, - order, - noise, - factor_2_dummy, - type_value, + order=order, + noise_reduction=noise, + is_class=type_value == "class", ) - if order is None or isinstance(order, int): - reg_points_matrix = _unique_rows_preserve_order(reg_points_matrix) + reg_points_matrix = np.full( + (max((boundary.size for boundary in model.boundaries), default=0), x_values.shape[1]), + np.nan, + dtype=np.float64, + ) + for col, boundary in enumerate(model.boundaries): + reg_points_matrix[: boundary.size, col] = boundary if order == "max" and n_best is None: n_best = 1 - nns_id_components = _find_interval_matrix(x_values, reg_points_matrix) - nns_ids = _join_ids(nns_id_components) - rpm, fitted_y, residuals = _rpm_and_fitted( - x_values, + nns_ids = model.ids + rpm = model.rpm + fitted_y, residuals = _fitted_from_rpm_ids( y_values, nns_ids, - noise, - order_is_numeric=order is None or isinstance(order, int), + rpm, class_mode=type_value == "class", ) @@ -182,8 +289,12 @@ def _render_m_reg(fitted: dict[str, NDArray[np.float64] | NDArray[np.str_]]) -> mask = np.isfinite(pos) & np.isfinite(neg) if mask.any(): ax.fill_between( - index[mask], neg[mask], pos[mask], - color=palette.PINK, alpha=palette.CI_ALPHA_REG, linewidth=0.0, + index[mask], + neg[mask], + pos[mask], + color=palette.PINK, + alpha=palette.CI_ALPHA_REG, + linewidth=0.0, ) ax.set_xlabel("Index") ax.set_ylabel("y (blue) y.hat (red)") @@ -300,16 +411,79 @@ def _unique_rows_preserve_order(values: NDArray[np.float64]) -> NDArray[np.float def _find_interval_matrix( x: NDArray[np.float64], reg_points_matrix: NDArray[np.float64], +) -> NDArray[np.int64]: + boundaries = tuple( + np.sort(reg_points_matrix[:, col][np.isfinite(reg_points_matrix[:, col])]) + for col in range(x.shape[1]) + ) + return _find_interval_matrix_from_boundaries(x, boundaries) + + +def _join_ids(components: NDArray[np.int64]) -> NDArray[np.str_]: + return np.asarray([".".join(str(int(v)) for v in row) for row in components], dtype=str) + + +def _find_interval_matrix_from_boundaries( + x: NDArray[np.float64], + boundaries: tuple[NDArray[np.float64], ...], ) -> NDArray[np.int64]: out = np.empty(x.shape, dtype=np.int64) - for col in range(x.shape[1]): - breaks = np.sort(reg_points_matrix[:, col][np.isfinite(reg_points_matrix[:, col])]) + for col, breaks in enumerate(boundaries): out[:, col] = np.searchsorted(breaks, x[:, col], side="right") + if breaks.size: + final_matches = x[:, col] == breaks[-1] + out[final_matches, col] = max(0, breaks.size - 1) return out -def _join_ids(components: NDArray[np.int64]) -> NDArray[np.str_]: - return np.asarray([".".join(str(int(v)) for v in row) for row in components], dtype=str) +def _rpm_from_ids( + x: NDArray[np.float64], + y: NDArray[np.float64], + nns_ids: NDArray[np.str_], + noise: NoiseReduction, + *, + order_is_numeric: bool, +) -> NDArray[np.float64]: + obs = np.arange(y.size) + sorted_order = np.lexsort((obs, nns_ids.astype(str))) + sorted_ids = nns_ids[sorted_order].astype(str) + unique_ids, first, inverse_sorted = np.unique( + sorted_ids, + return_index=True, + return_inverse=True, + ) + + sorted_matrix = np.column_stack((x[sorted_order], y[sorted_order])) + group_values = np.empty((unique_ids.size, x.shape[1] + 1), dtype=np.float64) + for group_index in range(unique_ids.size): + rows = sorted_matrix[inverse_sorted == group_index] + group_values[group_index] = _aggregate_rows(rows, noise, order_is_numeric) + return group_values[np.argsort(first)] + + +def _fitted_from_rpm_ids( + y: NDArray[np.float64], + nns_ids: NDArray[np.str_], + rpm: NDArray[np.float64], + *, + class_mode: bool, +) -> tuple[NDArray[np.float64], NDArray[np.float64]]: + ids_str = nns_ids.astype(str) + _, first = np.unique(ids_str, return_index=True) + rpm_ids = ids_str[np.sort(first)] + yhat_by_id = {group_id: rpm[index, -1] for index, group_id in enumerate(rpm_ids)} + initial_yhat = np.asarray([yhat_by_id[group_id] for group_id in ids_str], dtype=np.float64) + if class_mode: + initial_yhat = _round_clamp_classes(initial_yhat, y) + residuals = initial_yhat - y + bias = np.empty_like(residuals) + for group_id in rpm_ids: + mask = ids_str == group_id + bias[mask] = _gravity(residuals[mask]) + fitted_y = initial_yhat - bias + if class_mode: + fitted_y = _round_clamp_classes(fitted_y, y) + return fitted_y, fitted_y - y def _rpm_and_fitted( diff --git a/src/nns/regression.py b/src/nns/regression.py index 4e3ae8f1..6cad6caf 100644 --- a/src/nns/regression.py +++ b/src/nns/regression.py @@ -27,9 +27,12 @@ # Result shapes mirror installed R NNS.reg output names, including dotted keys, # hence the functional TypedDict syntax. + class RegPoints(TypedDict): x: NDArray[np.float64] y: NDArray[np.float64] + + """Consolidated regression points; also the whole result when ``multivariate_call=True``.""" DerivativeTable = TypedDict( @@ -70,13 +73,19 @@ class RegPoints(TypedDict): The ``conf.int.*`` columns are added only when ``confidence_interval`` is set. """ + class RegEquation(TypedDict): Variable: NDArray[np.str_] Coefficient: NDArray[np.float64] + + """Dimension-reduction synthetic-regressor weights (R's ``$equation``).""" + class RegXStar(TypedDict): x: NDArray[np.float64] + + """Synthetic dimension-reduction regressor (R's ``$x.star``).""" RegResult = TypedDict( @@ -204,7 +213,7 @@ def nns_reg( """ _warn_unsupported( return_values=return_values is not True, - ncores=ncores is not None, + ncores=ncores is not None and int(ncores) > 1, ) if dim_red_method is not None: @@ -229,8 +238,11 @@ def nns_reg( factor_levels=factor_levels, ) _maybe_render_reg( - result, plot=plot, plot_regions=plot_regions, - residual_plot=residual_plot, point_est=point_est, + result, + plot=plot, + plot_regions=plot_regions, + residual_plot=residual_plot, + point_est=point_est, ) return result @@ -269,6 +281,7 @@ def nns_reg( dist=dist, confidence_interval=confidence_interval, class_levels=class_levels, + ncores=ncores, plot=plot, residual_plot=residual_plot, ) @@ -303,8 +316,11 @@ def nns_reg( x_star=None, ) _maybe_render_reg( - result, plot=plot, plot_regions=plot_regions, - residual_plot=residual_plot, point_est=point_est, + result, + plot=plot, + plot_regions=plot_regions, + residual_plot=residual_plot, + point_est=point_est, ) return result diff --git a/src/nns/stack.py b/src/nns/stack.py index 7984d978..397375a3 100644 --- a/src/nns/stack.py +++ b/src/nns/stack.py @@ -16,13 +16,13 @@ from nns.categorical import _balance_class_training, _dense_factor_codes from nns.central_tendencies import nns_mode from nns.dependence import _gravity +from nns.multivariate_regression import _mreg_predict_path, _mreg_prepare_model from nns.regression import ( Order, RegResult, _expand_factor_predictors, _normalize_type, _prepare_y_values, - _r_minmax_columns, _round_clamp_classes, nns_reg, ) @@ -82,10 +82,12 @@ def nns_stack( random_seed: int | None = None, ) -> StackResult: """Port of R's deterministic numeric/classification NNS.stack orchestration.""" - _warn_unsupported(ncores=ncores is not None) # optimize_threshold defaults to True in R but the threshold search is not - # ported; status is R's console progress flag and NNS Python prints nothing. - del optimize_threshold, status + # ported. Keep status/ncores live: status reports coarse expensive stages and + # ncores is forwarded to the shared regression engine/native kernels. + del optimize_threshold + ncores_value = 1 if ncores is None else max(1, int(ncores)) + _warn_unsupported(ncores=ncores_value > 1) type_value = _normalize_type(type) if balance: type_value = "class" @@ -173,6 +175,8 @@ def nns_stack( ts_test=ts_test_value, pred_int=pred_int, type_value=type_value, + status=status, + ncores=ncores_value, ) method1_state = _evaluate_method1( x_train, @@ -193,6 +197,8 @@ def nns_stack( ts_test=ts_test_value, pred_int=pred_int, type_value=type_value, + status=status, + ncores=ncores_value, ) reg = method1_state.prediction @@ -282,6 +288,8 @@ def _evaluate_method2( ts_test: int | None, pred_int: float | None, type_value: str | None, + status: bool = False, + ncores: int = 1, ) -> _MethodState: n_rows, n_cols = x_train.shape if 2 not in methods or n_cols <= 1: @@ -296,6 +304,8 @@ def _evaluate_method2( relevant_vars = np.arange(n_cols, dtype=np.int64) for fold in range(1, folds + 1): + if status: + print(f"Method 2 fold {fold}/{folds}: calculating coefficients") train_idx, test_idx = _cv_split(n_rows, fold, cv_size, ts_test) cv_x_train = x_train[train_idx] cv_y_train = y_train[train_idx] @@ -303,6 +313,9 @@ def _evaluate_method2( cv_y_test = y_train[test_idx] cutoffs = _threshold_grid(cv_x_train, cv_y_train, dim_red_method, order, dist) + if status: + print(f"Method 2 fold {fold}/{folds}: generating {cutoffs.size} cumulative projections") + print(f"Method 2 fold {fold}/{folds}: evaluating {cutoffs.size} unique candidates") scores = np.empty(cutoffs.size, dtype=np.float64) class_thresholds = np.empty(cutoffs.size, dtype=np.float64) for idx, cutoff in enumerate(cutoffs): @@ -368,6 +381,7 @@ def _evaluate_method2( point_only=False, confidence_interval=pred_int, type=type_value, + ncores=ncores, ), ) fitted = cast(dict[str, NDArray[np.float64]], final_fit["Fitted.xy"]) @@ -382,6 +396,8 @@ def _evaluate_method2( final_pred_int = cast(dict[str, NDArray[np.float64]] | None, final_fit["pred.int"]) final_pred_int = _prediction_interval_or_point_estimate(final_pred_int, prediction) + if status: + print(f"Method 2 fold {folds}/{folds} complete") if stack and methods == (1, 2): train_star = cast(dict[str, NDArray[np.float64]], final_fit["x.star"])["x"] test_star = _xstar_for_points( @@ -429,6 +445,8 @@ def _evaluate_method1( ts_test: int | None, pred_int: float | None, type_value: str | None, + status: bool = False, + ncores: int = 1, ) -> _MethodState: if 1 not in methods: obj = math.inf if objective == "min" else -math.inf @@ -469,45 +487,37 @@ def _evaluate_method1( cv_x_train = cv_x_train[:, method2_state.relevant_vars] cv_x_test = cv_x_test[:, method2_state.relevant_vars] - setup = nns_reg( + if status: + print(f"Method 1 fold {fold}/{folds}: preparing design") + print(f"Method 1 fold {fold}/{folds}: building partitions and RPM") + kmax = min(l_value, cv_x_train.shape[0]) + print(f"Method 1 fold {fold}/{folds}: evaluating k = 1...{kmax}") + + model = _mreg_prepare_model( cv_x_train, cv_y_train, - point_est=cv_x_test, - n_best=1, order=order, - dist=dist, - point_only=False, - type=type_value, + noise_reduction="mode_class" if type_value == "class" else "off", + is_class=type_value == "class", ) - fitted = cast(dict[str, NDArray[np.float64]], setup["Fitted.xy"]) - yhat_vec = fitted["y.hat"] - setup_prediction = _as_prediction(setup["Point.est"], cv_x_test.shape[0]) - path_predictions = _distance_path_predictions( - cv_x_train, - yhat_vec, - cv_x_test, - min(l_value, cv_x_train.shape[0]), - ) - all_prediction = _distance_bulk_prediction( - cv_x_train, - yhat_vec, + if status: + print(f"Method 1 fold {fold}/{folds}: RPM rows = {model.rpm.shape[0]}") + candidate_counts = [max(1, min(int(k), model.rpm.shape[0])) for k in k_candidates] + path_by_k = _mreg_predict_path( + model, cv_x_test, - min(n_rows, cv_x_train.shape[0]), + k_values=candidate_counts, + is_class=type_value == "class", ) scores: list[float] = [] tested_ks: list[int] = [] class_thresholds: list[float] = [] - for k_value in k_candidates: - if k_value == 1: - predicted = setup_prediction - if type_value == "class" and np.any(np.isnan(predicted)): - predicted = predicted.copy() - predicted[np.isnan(predicted)] = float(np.nanmean(predicted)) - elif k_value <= path_predictions.shape[1]: - predicted = path_predictions[:, k_value - 1] - else: - predicted = all_prediction + for k_value, k_count in zip(k_candidates, candidate_counts, strict=True): + predicted = path_by_k[k_count] + if type_value == "class" and np.any(np.isnan(predicted)): + predicted = predicted.copy() + predicted[np.isnan(predicted)] = float(np.nanmean(predicted)) if type_value == "class": threshold_value = _classification_threshold( predicted, @@ -533,6 +543,8 @@ def _evaluate_method1( ) best_ks.append(tested_ks[best_index]) fold_scores.append(float(scores_arr[best_index])) + if status: + print(f"Method 1 fold {fold}/{folds} complete") best_k = int(_round_k_mode(np.asarray(best_ks, dtype=np.float64))) final_class_threshold = ( @@ -561,6 +573,7 @@ def _evaluate_method1( point_only=False, confidence_interval=pred_int, type=type_value, + ncores=ncores, ) fitted = cast(dict[str, NDArray[np.float64]], final_fit["Fitted.xy"]) prediction = _as_prediction(final_fit["Point.est"], x_test.shape[0]) @@ -771,9 +784,15 @@ def _xstar_for_points( if coef.size != test_x.shape[1]: fallback = cast(dict[str, NDArray[np.float64]], fit["x.star"])["x"] return np.full(test_x.shape[0], float(np.mean(fallback)), dtype=np.float64) - joint = np.vstack((test_x, train_x)) - norm = _r_minmax_columns(joint, zero_guard=True) - out = np.asarray(norm[: test_x.shape[0]] @ coef / active, dtype=np.float64) + train_min = np.min(train_x, axis=0) + train_max = np.max(train_x, axis=0) + denom = np.where(train_max == train_min, 1.0, train_max - train_min) + with np.errstate(over="ignore", invalid="ignore", divide="ignore"): + norm = (test_x - train_min[np.newaxis, :]) / denom[np.newaxis, :] + out = np.asarray(norm @ coef / active, dtype=np.float64) + out = np.nan_to_num( + out, nan=np.nan, posinf=np.finfo(np.float64).max, neginf=-np.finfo(np.float64).max + ) return _fill_nan_with_gravity(out) @@ -786,8 +805,8 @@ def _cv_split( if ts_test is not None: if ts_test < 1 or ts_test > n_rows: raise ValueError("ts_test must be in [1, n_rows].") - test_idx = np.arange(0, n_rows - ts_test, dtype=np.int64) - train_idx = np.arange(n_rows - ts_test, n_rows, dtype=np.int64) + train_idx = np.arange(0, n_rows - ts_test, dtype=np.int64) + test_idx = np.arange(n_rows - ts_test, n_rows, dtype=np.int64) if train_idx.size < 2: raise ValueError("ts_test leaves too few training rows.") return train_idx, test_idx diff --git a/tests/invariants/test_multivariate_regression.py b/tests/invariants/test_multivariate_regression.py index 9b05018c..5e992d96 100644 --- a/tests/invariants/test_multivariate_regression.py +++ b/tests/invariants/test_multivariate_regression.py @@ -3,7 +3,8 @@ import numpy as np import pytest -from nns import nns_m_reg +from nns import nns_m_reg, nns_reg +from nns.multivariate_regression import _find_interval_matrix_from_boundaries def test_nns_m_reg_shapes_and_bounds() -> None: @@ -107,3 +108,23 @@ def test_nns_m_reg_direct_factor_dummy_path_stays_rejected() -> None: with pytest.raises(NotImplementedError, match=r"prepare_factor_predictors"): nns_m_reg(x, y, factor_2_dummy=True) + + +def test_mreg_find_interval_final_boundary_matches_repaired_r() -> None: + x = np.array([[0.0], [1.0], [1.5], [2.0], [3.5], [4.0], [5.0]]) + ids = _find_interval_matrix_from_boundaries(x, (np.array([1.0, 2.0, 3.0, 4.0]),)) + np.testing.assert_array_equal(ids[:, 0], np.array([0, 1, 1, 2, 3, 3, 4])) + + +def test_nns_reg_and_nns_m_reg_share_multivariate_outputs() -> None: + x0 = np.linspace(-1.0, 1.0, 12) + x = np.column_stack((x0, x0**2)) + y = x0 + 0.5 * x0**2 + point = x[[0, 4, 8]] + + via_reg = nns_reg(x, y, point_est=point, n_best=2, point_only=False) + direct = nns_m_reg(x, y, point_est=point, n_best=2, point_only=False) + + np.testing.assert_allclose(via_reg["Point.est"], direct["Point.est"], atol=1e-12) + np.testing.assert_allclose(via_reg["RPM"]["y.hat"], direct["RPM"]["y.hat"], atol=1e-12) + np.testing.assert_array_equal(via_reg["Fitted.xy"]["NNS.ID"], direct["Fitted.xy"]["NNS.ID"]) diff --git a/tests/invariants/test_stack.py b/tests/invariants/test_stack.py index 3f340ec7..94d28f76 100644 --- a/tests/invariants/test_stack.py +++ b/tests/invariants/test_stack.py @@ -351,16 +351,16 @@ def test_nns_stack_ts_test_shape_and_determinism(method: tuple[int, ...]) -> Non np.testing.assert_allclose(first["stack"], second["stack"]) -def test_nns_stack_ts_test_split_matches_r_sizes() -> None: +def test_nns_stack_ts_test_split_uses_historical_prefix_for_training() -> None: train_idx, test_idx = _cv_split(40, fold=1, cv_size=0.25, ts_test=10) - assert train_idx.shape == (10,) - assert test_idx.shape == (30,) - np.testing.assert_array_equal(train_idx, np.arange(30, 40)) - np.testing.assert_array_equal(test_idx, np.arange(0, 30)) + assert train_idx.shape == (30,) + assert test_idx.shape == (10,) + np.testing.assert_array_equal(train_idx, np.arange(0, 30)) + np.testing.assert_array_equal(test_idx, np.arange(30, 40)) -@pytest.mark.parametrize("ts_test", [0, 1, 41]) +@pytest.mark.parametrize("ts_test", [0, 39, 41]) def test_nns_stack_invalid_ts_test_raises(ts_test: int) -> None: x = np.linspace(-2.0, 2.0, 40) variable = np.column_stack((x, np.sin(x), np.cos(x))) @@ -368,3 +368,47 @@ def test_nns_stack_invalid_ts_test_raises(ts_test: int) -> None: with pytest.raises(ValueError): nns_stack(variable, y, variable[:3], cv_size=0.25, folds=1, method=1, ts_test=ts_test) + + +def test_nns_stack_method1_scores_candidates_without_public_refits( + monkeypatch: pytest.MonkeyPatch, +) -> None: + x0 = np.linspace(-1.0, 1.0, 18) + variable = np.column_stack((x0, x0**2)) + y = x0 + 0.2 * x0**2 + import nns.stack as stack_mod + + calls = 0 + original = stack_mod.nns_reg + + def counting_nns_reg(*args: object, **kwargs: object) -> object: + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(stack_mod, "nns_reg", counting_nns_reg) + result = stack_mod.nns_stack(variable, y, variable[:3], folds=1, method=1) + + assert result["stack"].shape == (3,) + # Method 1 may use the public estimator for the final selected fit, but not + # once per candidate k. With n=18 the candidate grid has more than one k. + assert calls == 1 + + +def test_nns_stack_xstar_projection_scaling_is_batch_invariant() -> None: + import nns.stack as stack_mod + from nns.regression import nns_reg + + x0 = np.linspace(-1.0, 1.0, 24) + variable = np.column_stack((x0, np.sin(x0), np.cos(x0))) + y = 1.0 + x0 - 0.25 * np.sin(x0) + point = variable[5:6] + extreme = np.array([[10_000.0, -10_000.0, 10_000.0]]) + fit = nns_reg(variable, y, dim_red_method="cor", point_est=point, point_only=False) + + single = stack_mod._xstar_for_points(fit, variable, point, mixed_factor=False, raw_columns=0)[0] + batched = stack_mod._xstar_for_points( + fit, variable, np.vstack((point, extreme)), mixed_factor=False, raw_columns=0 + )[0] + + assert single == batched diff --git a/tests/invariants/test_stack_method1_pooled_oof.py b/tests/invariants/test_stack_method1_pooled_oof.py new file mode 100644 index 00000000..3d65367c --- /dev/null +++ b/tests/invariants/test_stack_method1_pooled_oof.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import numpy as np +import pytest + +from nns._stack_method1 import CandidateId, Method1FoldPredictions, select_method1_candidate + + +def _sse( + _candidate: CandidateId, + predicted: np.ndarray, + actual: np.ndarray, +) -> tuple[float, float]: + return float(np.sum((predicted - actual) ** 2)), 0.5 + + +def test_method1_requires_identical_complete_oof_coverage() -> None: + actual = np.arange(6, dtype=np.float64) + folds = [ + Method1FoldPredictions( + validation_idx=np.array([0, 1, 2], dtype=np.int64), + predictions={ + 1: np.array([0.0, 1.0, 2.0]), + 2: np.array([0.0, 1.0, 2.0]), + 3: np.array([0.0, 1.0, 2.0]), + "all": np.array([0.0, 1.0, 2.0]), + }, + ), + Method1FoldPredictions( + validation_idx=np.array([3, 4, 5], dtype=np.int64), + predictions={ + 1: np.array([3.0, 4.0, 5.0]), + 2: np.array([3.0, 4.0, 5.0]), + "all": np.array([3.0, 4.0, 5.0]), + }, + ), + ] + + result = select_method1_candidate( + n_obs=6, + actual=actual, + folds=folds, + local_candidates=[1, 2, 3], + evaluator=_sse, + objective="min", + ) + + assert 3 in result.excluded + assert 3 not in result.scores + np.testing.assert_array_equal(result.count_vectors[1], np.ones(6, dtype=np.int64)) + np.testing.assert_array_equal(result.count_vectors[2], result.count_vectors[1]) + assert not np.array_equal(result.count_vectors[3], result.count_vectors[1]) + + +def test_method1_empty_candidate_cannot_win_with_zero_sse() -> None: + actual = np.array([1.0, 2.0, 3.0, 4.0]) + folds = [ + Method1FoldPredictions( + validation_idx=np.arange(4, dtype=np.int64), + predictions={ + 1: np.array([1.1, 2.1, 3.1, 4.1]), + 2: np.array([1.2, 2.2, 3.2, 4.2]), + 3: np.full(4, np.nan), + "all": np.array([1.5, 2.5, 3.5, 4.5]), + }, + ) + ] + + result = select_method1_candidate( + n_obs=4, + actual=actual, + folds=folds, + local_candidates=[1, 2, 3], + evaluator=_sse, + objective="min", + ) + + assert 3 in result.excluded + assert 3 not in result.scores + assert result.selected == 1 + + +def test_method1_applies_early_stop_after_complete_pooled_scores() -> None: + actual = np.zeros(8, dtype=np.float64) + candidates = { + 1: np.full(8, 0.1), + 2: np.full(8, 0.2), + 3: np.full(8, 0.3), + 4: np.full(8, 0.4), + 5: np.zeros(8), + "all": np.full(8, 0.05), + } + folds = [ + Method1FoldPredictions( + validation_idx=np.arange(0, 4, dtype=np.int64), + predictions={key: value[:4] for key, value in candidates.items()}, + ), + Method1FoldPredictions( + validation_idx=np.arange(4, 8, dtype=np.int64), + predictions={key: value[4:] for key, value in candidates.items()}, + ), + ] + + result = select_method1_candidate( + n_obs=8, + actual=actual, + folds=folds, + local_candidates=[1, 2, 3, 4, 5], + evaluator=_sse, + objective="min", + ) + + assert result.stopping_k == 4 + assert result.eligible == (1, 2, 3, 4, "all") + assert 5 in result.excluded + assert result.selected == "all" + + +def test_method1_all_is_evaluated_even_after_local_stop() -> None: + actual = np.zeros(4, dtype=np.float64) + folds = [ + Method1FoldPredictions( + validation_idx=np.arange(4, dtype=np.int64), + predictions={ + 1: np.full(4, 0.2), + 2: np.full(4, 0.3), + 3: np.full(4, 0.4), + 4: np.full(4, 0.5), + "all": np.zeros(4), + }, + ) + ] + + result = select_method1_candidate( + n_obs=4, + actual=actual, + folds=folds, + local_candidates=[1, 2, 3, 4], + evaluator=_sse, + objective="min", + ) + + assert result.stopping_k == 4 + assert result.selected == "all" + assert result.scores["all"] == pytest.approx(0.0) + + +def test_method1_first_candidate_wins_exact_tie() -> None: + actual = np.zeros(3, dtype=np.float64) + folds = [ + Method1FoldPredictions( + validation_idx=np.arange(3, dtype=np.int64), + predictions={ + 1: np.ones(3), + 2: np.ones(3), + "all": np.ones(3), + }, + ) + ] + + result = select_method1_candidate( + n_obs=3, + actual=actual, + folds=folds, + local_candidates=[1, 2], + evaluator=_sse, + objective="min", + ) + + assert result.selected == 1 + + +def test_method1_rejects_missing_reference_coverage() -> None: + with pytest.raises(ValueError, match="candidate 1 has no OOF coverage"): + select_method1_candidate( + n_obs=2, + actual=np.zeros(2), + folds=[ + Method1FoldPredictions( + validation_idx=np.arange(2, dtype=np.int64), + predictions={1: np.full(2, np.nan), "all": np.zeros(2)}, + ) + ], + local_candidates=[1], + evaluator=_sse, + objective="min", + ) diff --git a/tests/invariants/test_stack_method1_public_integration.py b/tests/invariants/test_stack_method1_public_integration.py new file mode 100644 index 00000000..038ae4bd --- /dev/null +++ b/tests/invariants/test_stack_method1_public_integration.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import numpy as np +import pytest + +from nns.stack import nns_stack + + +def test_nns_stack_method1_uses_pooled_oof_selection_not_fold_mode( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Method 1 must score conceptual candidates after pooling all folds.""" + + import nns._stack_method1_runtime as runtime_mod + import nns.stack as stack_mod + + x = np.arange(12, dtype=np.float64) + variable = np.column_stack((x, x**2)) + y = np.zeros(12, dtype=np.float64) + + folds = [ + (np.arange(4, 12, dtype=np.int64), np.arange(0, 4, dtype=np.int64)), + (np.r_[0:4, 8:12].astype(np.int64), np.arange(4, 8, dtype=np.int64)), + (np.arange(0, 8, dtype=np.int64), np.arange(8, 12, dtype=np.int64)), + ] + split_iter = iter(folds) + monkeypatch.setattr(stack_mod, "_cv_split", lambda *_args, **_kwargs: next(split_iter)) + + class Model: + rpm = np.zeros((3, 3), dtype=np.float64) + + monkeypatch.setattr(stack_mod, "_mreg_prepare_model", lambda *_args, **_kwargs: Model()) + + paths = iter( + [ + {1: np.zeros(4), 2: np.full(4, 10.0), 3: np.full(4, 20.0)}, + {1: np.full(4, 2.0), 2: np.zeros(4), 3: np.full(4, 20.0)}, + {1: np.full(4, 2.0), 2: np.ones(4), 3: np.full(4, 20.0)}, + ] + ) + monkeypatch.setattr(stack_mod, "_mreg_predict_path", lambda *_args, **_kwargs: next(paths)) + monkeypatch.setattr( + runtime_mod, + "nns_distance_path_single_bulk", + lambda _rpm, points, _k, _type: np.full(np.asarray(points).shape[0], 50.0), + ) + + selected: list[int | str | None] = [] + + def fake_nns_reg(*_args: object, **kwargs: object) -> dict[str, object]: + selected.append(kwargs.get("n_best")) + n = np.asarray(kwargs["point_est"]).shape[0] + return { + "Point.est": np.zeros(n), + "Fitted.xy": {"y.hat": np.zeros(12), "y": np.zeros(12)}, + "pred.int": None, + } + + monkeypatch.setattr(stack_mod, "nns_reg", fake_nns_reg) + result = nns_stack(variable, y, variable[:2], folds=3, method=1) + + # Fold-local winners are 1, 2, 2, but pooled SSE chooses candidate 1. + assert result["NNS.reg.n.best"] == 1.0 + assert selected[-1] == 1 + + +def test_nns_stack_method1_all_is_not_encoded_as_training_row_count( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ALL remains conceptual until it resolves against the prepared RPM.""" + + import nns._stack_method1_runtime as runtime_mod + import nns.stack as stack_mod + + x = np.linspace(-1.0, 1.0, 16) + variable = np.column_stack((x, x**2)) + y = x + requested_paths: list[tuple[int, ...]] = [] + + class Model: + rpm = np.zeros((5, 3), dtype=np.float64) + + monkeypatch.setattr(stack_mod, "_mreg_prepare_model", lambda *_args, **_kwargs: Model()) + + def fake_path(*args: object, **kwargs: object) -> dict[int, np.ndarray]: + ks = tuple(int(k) for k in kwargs["k_values"]) + requested_paths.append(ks) + n = np.asarray(args[1]).shape[0] + return {k: np.zeros(n) for k in ks} + + monkeypatch.setattr(stack_mod, "_mreg_predict_path", fake_path) + monkeypatch.setattr( + runtime_mod, + "nns_distance_path_single_bulk", + lambda _rpm, points, _k, _type: np.zeros(np.asarray(points).shape[0]), + ) + nns_stack(variable, y, variable[:2], folds=1, method=1) + + assert requested_paths + assert 16 not in requested_paths[0] + assert max(requested_paths[0]) <= int(np.floor(np.sqrt(16))) diff --git a/tests/parity/check_repaired_r_docs.R b/tests/parity/check_repaired_r_docs.R new file mode 100644 index 00000000..a7d34b7c --- /dev/null +++ b/tests/parity/check_repaired_r_docs.R @@ -0,0 +1,43 @@ +#!/usr/bin/env Rscript +if (!file.exists("man/NNS.reg.Rd")) { + stop("Pinned R reference is missing man/NNS.reg.Rd") +} + +namespace <- readLines("NAMESPACE", warn = FALSE) + +if (!"export(NNS.reg)" %in% namespace) { + stop("Pinned R reference does not export NNS.reg") +} + +if (any(grepl("nns_reg_partition_points_fast", namespace, fixed = TRUE))) { + stop("Internal partition helper was incorrectly exported") +} + +if (any(grepl("nns_reg_univariate_fast", namespace, fixed = TRUE))) { + stop("Internal univariate helper was incorrectly exported") +} + +bad_docs <- list.files( + "man", + pattern = "partition_points_fast|univariate_fast", + full.names = TRUE +) + +if (length(bad_docs)) { + stop( + "Internal regression helper documentation was generated: ", + paste(bad_docs, collapse = ", ") + ) +} + +rd <- readLines("man/NNS.reg.Rd", warn = FALSE) + +if (!any(grepl("\\\\name\\{NNS.reg\\}", rd))) { + stop("man/NNS.reg.Rd has the wrong name") +} + +if (!any(grepl("\\\\alias\\{NNS.reg\\}", rd))) { + stop("man/NNS.reg.Rd has the wrong alias") +} + +cat("Pinned R documentation contract is valid\n") diff --git a/tests/parity/check_repaired_r_stack_invariants.R b/tests/parity/check_repaired_r_stack_invariants.R new file mode 100644 index 00000000..79d607a7 --- /dev/null +++ b/tests/parity/check_repaired_r_stack_invariants.R @@ -0,0 +1,383 @@ +#!/usr/bin/env Rscript +args <- commandArgs(trailingOnly = TRUE) +out_dir <- if (length(args) >= 1L) args[[1L]] else "artifacts/repaired-r-validation" +dir.create(out_dir, recursive = TRUE, showWarnings = FALSE) + +suppressPackageStartupMessages({ + library(jsonlite) + library(devtools) +}) + +options( + NNS.native.stack = FALSE, + NNS.native.mreg = FALSE, + NNS.native.univariate = FALSE +) + +devtools::load_all(getwd(), quiet = TRUE) + +`%||%` <- function(x, y) if (is.null(x)) y else x + +write_failure <- function(name, payload) { + write_json( + payload, + file.path(out_dir, paste0("stack-invariant-failure-", name, ".json")), + pretty = TRUE, + auto_unbox = TRUE, + digits = NA, + null = "null" + ) +} + +fail <- function(name, message, payload = list()) { + write_failure(name, c(list(message = message), payload)) + stop(message, call. = FALSE) +} + +assert_equal <- function(name, actual, expected, tolerance = 1e-12, + exact = FALSE, context = list()) { + ok <- if (exact) { + identical(as.vector(actual), as.vector(expected)) + } else { + isTRUE(all.equal( + as.numeric(actual), as.numeric(expected), + tolerance = tolerance, + check.attributes = FALSE + )) + } + + if (!ok) { + fail( + name, + paste0("R repaired stack invariant failed: ", name), + c( + list( + actual = as.vector(actual), + expected = as.vector(expected), + tolerance = tolerance, + exact = exact + ), + context + ) + ) + } +} + +check_scalar_clean <- function(result, fields) { + for (field in fields) { + value <- result[[field]] + if (is.null(value)) next + if (length(value) != 1L || !is.null(names(value))) { + fail( + paste0("scalar-clean-", field), + paste0("R stack scalar field is not an unnamed scalar: ", field), + list( + field = field, + value = value, + names = names(value), + length = length(value) + ) + ) + } + } +} + +field <- function(x, names_to_try, default = NULL) { + if (!is.list(x)) return(default) + for (name in names_to_try) { + if (!is.null(x[[name]])) return(x[[name]]) + } + default +} + +candidate_id <- function(candidate) { + value <- field(candidate, c( + "candidate", "candidate_id", "id", "k", "n.best", "n_best" + )) + if (is.null(value) || !length(value)) return(NA_character_) + as.character(value[[1L]]) +} + +with_stack_trace <- function(expr) { + trace_env <- new.env(parent = emptyenv()) + old <- options(NNS.stack.trace.env = trace_env) + on.exit(options(old), add = TRUE) + + result <- force(expr) + trace <- trace_env$method1 + + if (is.null(trace)) { + fail( + "method1-trace-missing", + paste( + "NNS.stack did not write Method 1 internals to", + "options(NNS.stack.trace.env = ).", + "Fixture generation is blocked until the R reference exposes", + "the hidden trace without changing the public return value." + ), + list(result_names = names(result)) + ) + } + + list(result = result, trace = trace) +} + +normalize_trace_folds <- function(trace) { + folds <- field(trace, c("folds", "fold", "fold_trace", "fold_traces")) + if (is.null(folds) && is.list(trace) && length(trace) && + all(vapply(trace, is.list, logical(1L)))) { + folds <- trace + } + if (is.null(folds) || !length(folds)) { + fail( + "method1-trace-no-folds", + "Method 1 trace does not contain fold records.", + list(trace_names = names(trace)) + ) + } + folds +} + +check_internal_candidates <- function() { + set.seed(131) + n <- 500L + grid <- seq(-2, 2, length.out = n) + X <- cbind( + x1 = grid, + x2 = sin(grid), + x3 = cos(grid), + x4 = rep(c(-1, 0, 1, 0), length.out = n), + x5 = grid^2 + ) + y <- X[, 1L] + 0.5 * X[, 2L] - X[, 3L] + 0.1 * X[, 4L] + + run <- with_stack_trace(NNS.stack( + IVs.train = X, + DV.train = y, + IVs.test = X[1:5, , drop = FALSE], + method = 1, + ts.test = 50, + folds = 3, + status = FALSE, + ncores = 1 + )) + + result <- run$result + trace <- run$trace + folds <- normalize_trace_folds(trace) + + pooled_counts <- list() + pooled_scores <- list() + pooled_predictions <- list() + pooled_actuals <- list() + + for (fold_idx in seq_along(folds)) { + fold <- folds[[fold_idx]] + train_x <- field(fold, c( + "train_design", "train_x", "fold_train_x", "encoded_train_x", "x_train" + )) + train_y <- field(fold, c( + "train_y_fit", "train_y", "fold_train_y", "y_train" + )) + validation_x <- field(fold, c( + "valid_design", "validation_x", "fold_validation_x", + "encoded_validation_x", "x_validation", "x_valid" + )) + validation_y <- field(fold, c( + "validation_y", "fold_validation_y", "y_validation", "y_valid" + )) + response_offset <- as.numeric(field( + fold, c("response_offset", "offset"), default = 0 + )) + order_value <- field(fold, c("order"), default = NULL) + dist_value <- field(fold, c("dist", "distance"), default = "L2") + candidates <- field(fold, c( + "candidates", "candidate_predictions", "method1_candidates" + )) + + if (is.null(validation_y)) { + valid_idx <- field(fold, c("valid_idx", "validation_idx")) + if (!is.null(valid_idx)) validation_y <- y[as.integer(valid_idx)] + } + + required <- list( + train_x = train_x, + train_y = train_y, + validation_x = validation_x, + validation_y = validation_y, + candidates = candidates + ) + missing <- names(required)[vapply(required, is.null, logical(1L))] + if (length(missing)) { + fail( + "method1-trace-incomplete", + "Method 1 trace is missing fold inputs or candidate predictions.", + list(fold = fold_idx, missing = missing, fold_names = names(fold)) + ) + } + + for (candidate in candidates) { + id <- candidate_id(candidate) + prediction <- field(candidate, c( + "prediction", "predictions", "point_est", "point.est", "Point.est" + )) + eligible <- isTRUE(field(candidate, c( + "eligible", "complete", "scored" + ), default = TRUE)) + score <- field(candidate, c("score", "objective", "OBJfn", "sse"), + default = NA_real_) + + if (is.na(id) || is.null(prediction)) { + fail( + "method1-candidate-trace-invalid", + "A traced Method 1 candidate lacks an ID or prediction vector.", + list(fold = fold_idx, candidate_names = names(candidate)) + ) + } + + if (!eligible) { + if (length(score) && !all(is.na(score))) { + fail( + "method1-excluded-candidate-scored", + "A partial or excluded candidate received an objective.", + list(fold = fold_idx, candidate = id, score = score) + ) + } + next + } + + if (!length(prediction) || !length(validation_y)) { + fail( + "method1-empty-vector-scored", + "Empty predicted/actual vectors cannot produce an objective.", + list(fold = fold_idx, candidate = id, score = score) + ) + } + + direct_n_best <- if (tolower(id) == "all") "all" else as.integer(id) + direct <- NNS.reg( + train_x, + train_y, + point.est = validation_x, + n.best = direct_n_best, + order = order_value, + dist = dist_value, + plot = FALSE, + residual.plot = FALSE, + factor.2.dummy = FALSE, + point.only = TRUE, + ncores = 1 + )$Point.est + direct <- as.numeric(direct) - response_offset + + assert_equal( + paste0("method1-internal-vs-direct-fold-", fold_idx, + "-candidate-", id), + prediction, + direct, + tolerance = 1e-12, + context = list( + fold = fold_idx, + candidate = id, + internal_prediction = as.numeric(prediction), + direct_prediction = direct, + max_abs_diff = max(abs(as.numeric(prediction) - direct)) + ) + ) + + pooled_counts[[id]] <- (pooled_counts[[id]] %||% 0L) + length(prediction) + pooled_predictions[[id]] <- c( + pooled_predictions[[id]] %||% numeric(), as.numeric(prediction) + ) + pooled_actuals[[id]] <- c( + pooled_actuals[[id]] %||% numeric(), as.numeric(validation_y) + ) + pooled_scores[[id]] <- sum( + (pooled_predictions[[id]] - pooled_actuals[[id]])^2 + ) + } + } + + if (is.null(pooled_counts[["1"]])) { + fail( + "method1-k1-missing", + "Candidate k=1 must define the reference OOF coverage.", + list(counts = pooled_counts) + ) + } + + reference_count <- pooled_counts[["1"]] + ordinary_ids <- setdiff(names(pooled_counts), "all") + mismatched <- ordinary_ids[vapply( + ordinary_ids, + function(id) pooled_counts[[id]] != reference_count, + logical(1L) + )] + if (length(mismatched)) { + fail( + "method1-ordinary-count-vector", + "Eligible ordinary candidates do not share k=1 OOF coverage.", + list( + counts = pooled_counts, + reference_count = reference_count, + mismatched = mismatched + ) + ) + } + + if (is.null(pooled_counts[["all"]]) || + pooled_counts[["all"]] != reference_count) { + fail( + "method1-all-count", + "ALL does not have complete OOF coverage.", + list(counts = pooled_counts, reference_count = reference_count) + ) + } + + if (any(!is.finite(unlist(pooled_scores)))) { + fail( + "method1-pooled-scores", + "Pooled Method 1 candidate scores must be finite.", + list(scores = pooled_scores) + ) + } + + proof <- list( + reference_count = reference_count, + candidate_counts = pooled_counts, + candidate_scores = pooled_scores, + eligible_candidates = field(trace, c("eligible_candidates", "eligible_ids"), + default = names(pooled_counts)), + excluded_candidates = field(trace, c("excluded_candidates", "excluded_ids"), + default = character()), + stopping_k = field(trace, c("stopping_k", "stop_k"), default = NULL), + selected_candidate = field( + trace, + c("selected_candidate", "selected_id"), + default = result$NNS.reg.n.best + ) + ) + + write_json( + proof, + file.path(out_dir, "stack-invariant-method1-coverage-proof.json"), + pretty = TRUE, + auto_unbox = TRUE, + digits = NA, + null = "null" + ) + + check_scalar_clean( + result, + c( + "OBJfn.reg", "NNS.reg.n.best", "OBJfn.dim.red", + "NNS.dim.red.threshold", "probability.threshold" + ) + ) +} + +check_internal_candidates() +cat(paste( + "Repaired R internal Method 1 candidate, complete-OOF coverage,", + "ALL, and scalar invariants passed\n" +)) diff --git a/tests/parity/fixtures/repaired_r_13_1/.gitkeep b/tests/parity/fixtures/repaired_r_13_1/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/parity/fixtures/repaired_r_13_1_21be6d92/.gitkeep b/tests/parity/fixtures/repaired_r_13_1_21be6d92/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/parity/fixtures/repaired_r_13_1_54c98418/.gitkeep b/tests/parity/fixtures/repaired_r_13_1_54c98418/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/parity/fixtures/repaired_r_13_1_c13fb4f/.gitkeep b/tests/parity/fixtures/repaired_r_13_1_c13fb4f/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/parity/generate_repaired_r_fixtures.R b/tests/parity/generate_repaired_r_fixtures.R new file mode 100644 index 00000000..53f41b27 --- /dev/null +++ b/tests/parity/generate_repaired_r_fixtures.R @@ -0,0 +1,358 @@ +#!/usr/bin/env Rscript +args <- commandArgs(trailingOnly = TRUE) +get_arg <- function(name, default = NULL) { + flag <- paste0("--", name) + idx <- match(flag, args) + if (is.na(idx) || idx == length(args)) return(default) + args[[idx + 1L]] +} + +r_repo <- normalizePath(get_arg("r-repo", "../NNS-r"), mustWork = TRUE) +out_dir <- get_arg("out", "tests/parity/fixtures/repaired_r_13_1_21be6d92") +r_commit <- get_arg("commit", NA_character_) +dir.create(out_dir, recursive = TRUE, showWarnings = FALSE) + +suppressPackageStartupMessages({ + library(jsonlite) + library(devtools) + library(digest) +}) + +options( + NNS.native.stack = FALSE, + NNS.native.mreg = FALSE, + NNS.native.univariate = FALSE +) + +devtools::load_all(r_repo, quiet = TRUE) + +pkg_desc <- desc::desc(file.path(r_repo, "DESCRIPTION")) +metadata <- list( + fixture_schema_version = "repaired_r_13_1_21be6d92", + generated_at = format(Sys.time(), "%Y-%m-%dT%H:%M:%SZ", tz = "UTC"), + r_repository = "OVVO-Financial/NNS", + r_commit_sha = r_commit, + nns_version = pkg_desc$get("Version"), + r_version = paste(R.version$major, R.version$minor, sep = "."), + platform = R.version$platform, + os = Sys.info()[["sysname"]], + native_reference_options = list( + NNS.native.stack = getOption("NNS.native.stack"), + NNS.native.mreg = getOption("NNS.native.mreg"), + NNS.native.univariate = getOption("NNS.native.univariate") + ) +) + +write_json(metadata, file.path(out_dir, "metadata.json"), pretty = TRUE, auto_unbox = TRUE) +manifest <- list( + schema_version = 1L, + r_repository = metadata$r_repository, + r_commit = metadata$r_commit_sha, + nns_version = metadata$nns_version, + reference_backend = list(stack = FALSE, mreg = FALSE, univariate = FALSE) +) +write_json(manifest, file.path(out_dir, "manifest.json"), pretty = TRUE, auto_unbox = TRUE) + +case_rows <- list() +with_checksums <- function(case) { + case$input_checksum <- digest(case$input, algo = "sha256", serialize = TRUE) + case$output_checksum <- digest(case$output, algo = "sha256", serialize = TRUE) + case +} + +capture_part_case <- function(name, x, y = NULL, order = NULL, type = NULL, + noise.reduction = "mean", obs.req = NULL) { + call_args <- list( + x = x, y = y, order = order, type = type, + noise.reduction = noise.reduction, obs.req = obs.req + ) + result <- do.call(NNS.part, call_args[!vapply(call_args, is.null, logical(1L))]) + with_checksums(list( + name = name, + kind = "part", + input = list(x = x, y = y), + args = list(order = order, type = type, + noise.reduction = noise.reduction, obs.req = obs.req), + output = result + )) +} + +capture_part_error_case <- function(name, x, y = NULL, order = NULL, + type = NULL, obs.req = NULL) { + error <- tryCatch({ + call_args <- list(x = x, y = y, order = order, type = type, obs.req = obs.req) + do.call(NNS.part, call_args[!vapply(call_args, is.null, logical(1L))]) + NULL + }, error = function(e) conditionMessage(e)) + with_checksums(list( + name = name, + kind = "part", + input = list(x = x, y = y), + args = list(order = order, type = type, obs.req = obs.req), + output = list(error = error) + )) +} + +capture_reg_case <- function(name, x, y, point = NULL, type = NULL, + order = NULL, n.best = NULL, smooth = FALSE, + pred.int = NULL) { + result <- NNS.reg( + x = x, + y = y, + point.est = point, + type = type, + order = order, + n.best = n.best, + smooth = smooth, + pred.int = pred.int, + plot = FALSE, + residual.plot = FALSE, + ncores = 1 + ) + with_checksums(list( + name = name, + kind = "reg", + input = list( + x = unclass(x), y = unclass(y), + point = if (is.null(point)) NULL else unclass(point) + ), + args = list(type = type, order = order, n.best = n.best, + smooth = smooth, pred.int = pred.int), + output = result + )) +} + +capture_mreg_case <- function(name, x, y, point = NULL, type = NULL, + order = NULL, n.best = NULL) { + result <- NNS.M.reg( + X_n = x, + Y = y, + point.est = point, + type = type, + order = order, + n.best = n.best, + plot = FALSE, + residual.plot = FALSE, + ncores = 1 + ) + with_checksums(list( + name = name, + kind = "mreg", + input = list( + x = unclass(x), y = unclass(y), + point = if (is.null(point)) NULL else unclass(point) + ), + args = list(type = type, order = order, n.best = n.best), + output = result + )) +} + +capture_stack_case <- function(name, x, y, point = NULL, + method = c(1, 2), type = NULL, + ts.test = NULL, balance = FALSE, + pred.int = NULL) { + result <- NNS.stack( + IVs.train = x, + DV.train = y, + IVs.test = point, + method = method, + type = type, + ts.test = ts.test, + balance = balance, + pred.int = pred.int, + folds = 1, + status = FALSE, + ncores = 1 + ) + with_checksums(list( + name = name, + kind = "stack", + input = list( + x = unclass(x), y = unclass(y), + point = if (is.null(point)) NULL else unclass(point) + ), + args = list(method = method, type = type, ts.test = ts.test, + balance = balance, pred.int = pred.int), + output = result + )) +} + +capture_boost_case <- function(name, x, y, point = NULL, type = NULL, + ts.test = NULL, pred.int = NULL) { + result <- NNS.boost( + IVs.train = x, + DV.train = y, + IVs.test = point, + type = type, + ts.test = ts.test, + pred.int = pred.int, + learner.trials = 10, + status = FALSE, + ncores = 1 + ) + with_checksums(list( + name = name, + kind = "boost", + input = list( + x = unclass(x), y = unclass(y), + point = if (is.null(point)) NULL else unclass(point) + ), + args = list(type = type, ts.test = ts.test, + pred.int = pred.int, learner.trials = 10), + output = result + )) +} + +capture_var_case <- function(name, variables, h = 3, tau = 1, + dim.red.method = "cor") { + result <- NNS.VAR( + variables = variables, + h = h, + tau = tau, + dim.red.method = dim.red.method, + status = FALSE, + ncores = 1 + ) + with_checksums(list( + name = name, + kind = "var", + input = list(variables = unclass(variables)), + args = list(h = h, tau = tau, dim.red.method = dim.red.method), + output = result + )) +} + +part_x <- c(1, 1, 2, 3, 4, NA, NaN, Inf) +part_y <- c(2, 4, 6, 8, 10, 12, 14, 16) +case_rows[[length(case_rows) + 1L]] <- capture_part_case( + "part_default", part_x, part_y, order = NULL, type = NULL, + noise.reduction = "mean" +) +case_rows[[length(case_rows) + 1L]] <- capture_part_case( + "part_numeric_order", part_x, part_y, order = 2, type = NULL, + noise.reduction = "median" +) +case_rows[[length(case_rows) + 1L]] <- capture_part_case( + "part_order_max", part_x, part_y, order = "max", type = NULL, + noise.reduction = "off" +) +case_rows[[length(case_rows) + 1L]] <- capture_part_case( + "part_order_max_xonly", part_x, part_y, order = "max", type = "XONLY", + noise.reduction = "mean" +) +case_rows[[length(case_rows) + 1L]] <- capture_part_case( + "part_mode", c(1, 1, 2, 2, 3), c(4, 4, 5, 6, 6), + order = "max", type = "XONLY", noise.reduction = "mode" +) +case_rows[[length(case_rows) + 1L]] <- capture_part_case( + "part_mode_class", c(1, 1, 2, 2, 3), c(10, 20, 20, 10, 10), + order = "max", type = "XONLY", noise.reduction = "mode.class" +) +case_rows[[length(case_rows) + 1L]] <- capture_part_error_case( + "part_invalid_type", part_x, part_y, order = 1, type = "INVALID" +) +case_rows[[length(case_rows) + 1L]] <- capture_part_error_case( + "part_invalid_order", part_x, part_y, order = 0, type = NULL +) +case_rows[[length(case_rows) + 1L]] <- capture_part_error_case( + "part_invalid_obs_req", part_x, part_y, order = 1, + type = NULL, obs.req = 0 +) + +reg_x <- seq(-3, 3, length.out = 18) +reg_y <- reg_x^3 - reg_x +case_rows[[length(case_rows) + 1L]] <- capture_reg_case( + "reg_default", reg_x, reg_y, point = c(-2.5, 0.25, 3.5), order = NULL +) +case_rows[[length(case_rows) + 1L]] <- capture_reg_case( + "reg_integer_order", reg_x, reg_y, point = c(-2.5, 0.25, 3.5), order = 2 +) +case_rows[[length(case_rows) + 1L]] <- capture_reg_case( + "reg_order_max", c(reg_x, reg_x[5L]), c(reg_y, reg_y[5L] + 1), + point = c(-2.5, 0.25, 3.5), order = "max" +) +case_rows[[length(case_rows) + 1L]] <- capture_reg_case( + "reg_smooth", reg_x, reg_y + sin(reg_x), + point = c(-2.5, 0.25, 3.5), smooth = TRUE +) +case_rows[[length(case_rows) + 1L]] <- capture_reg_case( + "reg_class_nonconsecutive", reg_x, + ifelse(reg_x < -1, 10, ifelse(reg_x > 1, 30, 20)), + point = c(-2, 0, 2), type = "CLASS" +) +case_rows[[length(case_rows) + 1L]] <- capture_reg_case( + "reg_factor_response", reg_x, + factor(ifelse(reg_x < 0, "down", "up"), levels = c("up", "down")), + point = c(-2, 2), type = "CLASS" +) +case_rows[[length(case_rows) + 1L]] <- capture_reg_case( + "reg_pred_int", reg_x, reg_y, + point = c(-2.5, 0.25, 3.5), pred.int = 0.95 +) + +x0 <- seq(-2, 2, length.out = 24) +X <- cbind(x0, sin(x0), cos(x0)) +y <- x0 + sin(x0) +case_rows[[length(case_rows) + 1L]] <- capture_mreg_case( + "numeric_l2_default", X, y, X[1:4, ], order = NULL, n.best = NULL +) +case_rows[[length(case_rows) + 1L]] <- capture_mreg_case( + "numeric_order_max", X, y, X[1:4, ], order = "max", n.best = 1 +) +case_rows[[length(case_rows) + 1L]] <- capture_mreg_case( + "rightmost_boundary", + matrix(c(0, 1, 1.5, 2, 3.5, 4, 5), ncol = 1), + c(0, 1, 1, 2, 3, 3, 4), + matrix(c(1, 4, 5), ncol = 1), + order = "max", n.best = 1 +) +classes <- ifelse(x0 < -0.5, 1, ifelse(x0 > 0.75, 3, 2)) +case_rows[[length(case_rows) + 1L]] <- capture_mreg_case( + "multiclass", X, classes, X[1:4, ], + type = "CLASS", order = 1, n.best = 1 +) +case_rows[[length(case_rows) + 1L]] <- capture_stack_case( + "stack_method1_regression", X, y, X[1:4, ], method = 1 +) +case_rows[[length(case_rows) + 1L]] <- capture_stack_case( + "stack_method12_ts", X, y, X[1:4, ], + method = c(1, 2), ts.test = 5 +) +case_rows[[length(case_rows) + 1L]] <- capture_stack_case( + "stack_classification", X, classes, X[1:4, ], + method = c(1, 2), type = "CLASS" +) +case_rows[[length(case_rows) + 1L]] <- capture_stack_case( + "stack_pred_int", X, y, X[1:4, ], + method = c(1, 2), pred.int = 0.95 +) +case_rows[[length(case_rows) + 1L]] <- capture_boost_case( + "boost_numeric", X, y, X[1:4, ] +) +case_rows[[length(case_rows) + 1L]] <- capture_boost_case( + "boost_ts", X, y, X[1:4, ], ts.test = 5 +) +case_rows[[length(case_rows) + 1L]] <- capture_boost_case( + "boost_class_pred_int", X, classes, X[1:4, ], + type = "CLASS", pred.int = 0.95 +) +variables <- cbind(seq(-2, 17, by = 1), seq(1, 39, by = 2)) +case_rows[[length(case_rows) + 1L]] <- capture_var_case( + "var_cor_tau1", variables, h = 3, tau = 1, dim.red.method = "cor" +) +variables_missing <- variables +variables_missing[5, 1] <- NA +variables_missing[nrow(variables_missing), 2] <- NA +case_rows[[length(case_rows) + 1L]] <- capture_var_case( + "var_cor_missing", variables_missing, h = 3, tau = 2, + dim.red.method = "cor" +) + +write_json( + list(metadata = metadata, manifest = manifest, cases = case_rows), + file.path(out_dir, "fixtures.json"), + pretty = TRUE, + auto_unbox = TRUE, + digits = NA +) +cat("Generated repaired R fixtures in ", out_dir, "\n", sep = "") diff --git a/tests/parity/verify_repaired_r_fixtures.py b/tests/parity/verify_repaired_r_fixtures.py new file mode 100644 index 00000000..60a087f5 --- /dev/null +++ b/tests/parity/verify_repaired_r_fixtures.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import json +import sys +from pathlib import Path + +EXPECTED_SCHEMA = "repaired_r_13_1_21be6d92" +EXPECTED_REPOSITORY = "OVVO-Financial/NNS" +EXPECTED_R_SHA = "21be6d92d8ad23f0848191b094aded0dd6df8f74" +REQUIRED_FAMILIES = {"part", "reg", "mreg", "stack", "boost", "var"} + +REQUIRED_METADATA = { + "fixture_schema_version", + "generated_at", + "r_repository", + "r_commit_sha", + "nns_version", + "r_version", + "platform", + "os", + "native_reference_options", +} + + +def main() -> int: + fixture_dir = ( + Path(sys.argv[1]) + if len(sys.argv) > 1 + else Path("tests/parity/fixtures/repaired_r_13_1_21be6d92") + ) + metadata_path = fixture_dir / "metadata.json" + fixtures_path = fixture_dir / "fixtures.json" + missing_files = [str(path) for path in (metadata_path, fixtures_path) if not path.exists()] + if missing_files: + raise SystemExit(f"Missing repaired fixture files: {', '.join(missing_files)}") + metadata = json.loads(metadata_path.read_text()) + missing_keys = sorted(REQUIRED_METADATA - set(metadata)) + if missing_keys: + raise SystemExit(f"metadata.json missing keys: {', '.join(missing_keys)}") + if metadata.get("fixture_schema_version") != EXPECTED_SCHEMA: + raise SystemExit(f"unexpected fixture schema: {metadata.get('fixture_schema_version')!r}") + if metadata.get("r_repository") != EXPECTED_REPOSITORY: + raise SystemExit(f"unexpected R repository: {metadata.get('r_repository')!r}") + if metadata.get("r_commit_sha") != EXPECTED_R_SHA: + raise SystemExit(f"unexpected R commit: {metadata.get('r_commit_sha')!r}") + options = metadata["native_reference_options"] + expected_false = ["NNS.native.stack", "NNS.native.mreg", "NNS.native.univariate"] + bad_options = [name for name in expected_false if options.get(name) is not False] + if bad_options: + raise SystemExit( + "semantic fixtures must force reference backends: " + ", ".join(bad_options) + ) + fixtures = json.loads(fixtures_path.read_text()) + cases = fixtures.get("cases", []) + if not cases: + raise SystemExit("fixtures.json contains no cases") + names = [case.get("name") for case in cases] + if len(names) != len(set(names)): + raise SystemExit("fixture case names must be unique") + families = {str(case.get("kind")) for case in cases} + missing_families = REQUIRED_FAMILIES - families + if missing_families: + raise SystemExit( + "missing required fixture families: " + ", ".join(sorted(missing_families)) + ) + missing_checksums = [ + case.get("name") + for case in cases + if not case.get("input_checksum") or not case.get("output_checksum") + ] + if missing_checksums: + raise SystemExit( + "fixture cases missing checksums: " + ", ".join(map(str, missing_checksums)) + ) + print(f"Verified {len(cases)} repaired R fixture cases in {fixture_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())