From 4abe7ef384dc819d13ffc4a7580991a020eb90f6 Mon Sep 17 00:00:00 2001 From: lmoresi Date: Thu, 3 Sep 2026 15:19:42 -0700 Subject: [PATCH] Strip every launcher's variables before the serial reference child (#675) test_parallel_matches_serial_bit_identical spawns a serial reference subprocess from inside an mpirun and scrubs the launcher's variables so the child does not try to join the parent's job. Its scrub list named the Open MPI family only: OMPI_, PMIX_, PRTE_, PRTERUN_, OPAL_PREFIX. MPICH advertises itself through PMI_*. On the Linux CI runner the child therefore inherited PMI_FD and tried to join a job it was not part of: [cli_0]: PMIU_write error; fd=9 buf=:cmd=init pmi_version=1 system msg for write_line failure : Bad file descriptor Abort: internal_Init_thread ... PMI_Init returned -1 assert 15 == 0 macOS runs Open MPI, where that list IS complete, so this passed on one machine and failed on the other with nothing to do with the operating system. It had never been caught because the file matches no glob in either test script and had never run in CI at all. The complete list already existed thirty lines away, in serial_reference, and this test hand-rolled a copy that drifted. It now imports the shared tuple, which gains PRTE_/PRTERUN_/OPAL_ so the union is strictly wider than either copy was -- swapping to the shared list unchanged would have narrowed Open MPI 5 coverage, since PRRTE is a separate family from OMPI_. test_0873_adapt_collective_stop_mpi was only ever collateral: it passes in isolation, and failed downstream of this abort, which also produced the divergence that hung the batch for 76 minutes (rank 0 in _from_plexh5, rank 1 in barrier). Verified on the Linux/MPICH runner, both files, whole: 8 passed in 3.81s. Open MPI 5.0.10 locally: test_0855 4 passed, and test_1069, which uses serial_reference directly, 7 passed 1 skipped. Underworld development team with AI support from Claude Code Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01E87Q7KrpapxeQiLD1RiNXv --- tests/parallel/serial_reference.py | 8 +++++++- .../test_0855_mesh_smoothing_parallel.py | 20 ++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/parallel/serial_reference.py b/tests/parallel/serial_reference.py index ae147c3a2..90c366a1b 100644 --- a/tests/parallel/serial_reference.py +++ b/tests/parallel/serial_reference.py @@ -42,7 +42,13 @@ import underworld3 as uw # Launcher variables that would make a spawned singleton try to join the parent job. -_MPI_ENV_PREFIXES = ("OMPI_", "PMIX_", "PMI_", "MPI_", "HYDRA_", "I_MPI_", "SLURM_") +# Every implementation we run under is named here, because a list that covers only +# the one on your desk passes locally and fails on the other one: a copy of this +# naming just the Open MPI family passed under Open MPI and aborted the child's +# MPI_Init under MPICH, whose variables are PMI_* (#675). PRTE_/PRTERUN_/OPAL_ are +# Open MPI 5's runtime, which is a separate family again from OMPI_. +_MPI_ENV_PREFIXES = ("OMPI_", "PMIX_", "PMI_", "MPI_", "HYDRA_", "I_MPI_", "SLURM_", + "PRTE_", "PRTERUN_", "OPAL_") _CACHE = {} diff --git a/tests/parallel/test_0855_mesh_smoothing_parallel.py b/tests/parallel/test_0855_mesh_smoothing_parallel.py index 6f98f22c8..2f11ca5cf 100644 --- a/tests/parallel/test_0855_mesh_smoothing_parallel.py +++ b/tests/parallel/test_0855_mesh_smoothing_parallel.py @@ -36,6 +36,8 @@ import underworld3 as uw from underworld3.meshing import smooth_mesh_interior +from serial_reference import _MPI_ENV_PREFIXES + pytestmark = [pytest.mark.mpi(min_size=2), pytest.mark.timeout(120)] @@ -248,21 +250,21 @@ def test_parallel_matches_serial_bit_identical(): comm = MPI.COMM_WORLD rank = comm.rank - # 1. Compute the serial reference (rank 0 only) in a clean - # subprocess. We strip MPI/PMIX/PRTE env vars first so the - # subprocess's PETSc doesn't try to attach to this mpirun's - # MPI world (which would deadlock both processes). + # 1. Compute the serial reference (rank 0 only) in a clean subprocess. The + # launcher's variables are stripped first, or the child's MPI_Init tries to + # join THIS mpirun's job and aborts on a descriptor it does not own. + # + # The prefix list is shared with serial_reference rather than restated here: + # a local copy that named only the Open MPI family passed on Open MPI and + # failed under MPICH, whose variables are PMI_* (#675). One list, so a + # launcher that is missing from it is missing everywhere and gets noticed. ref_path = None if rank == 0: tmpdir = tempfile.mkdtemp(prefix="winslow_ref_") ref_path = os.path.join(tmpdir, "ref.npz") clean_env = { k: v for k, v in os.environ.items() - if not (k.startswith("OMPI_") - or k.startswith("PMIX_") - or k.startswith("PRTE_") - or k.startswith("PRTERUN_") - or k == "OPAL_PREFIX") + if not k.startswith(_MPI_ENV_PREFIXES) } proc = subprocess.run( [sys.executable, "-c",