From c91413f32148198e96eda9e3f1c8e69a74141898 Mon Sep 17 00:00:00 2001 From: Ryan Thomas Wollaeger Date: Sat, 1 Aug 2026 20:33:52 -0600 Subject: [PATCH 1/5] Update gray mode for compatibility with latest singularity-opac. + Bump singularity-opac submodule. + Update oapcity type names constructors and API calls. + Use 1-group construction for gray mode in mcblock. + Access hard-coded group 0 for gray mode runs. + Fix mcblock field opacity update: use optional (missing) gmode. + Ignore dirty submodules (following A. Dempsey's artemis change). --- .gitmodules | 15 ++++++++------ external/singularity-opac | 2 +- src/jaybenne/jaybenne.cpp | 30 +++++++++++++++------------ src/jaybenne/sourcing.cpp | 4 +++- src/mcblock/mcblock.cpp | 43 ++++++++++++++++++++++++++++----------- src/mcblock/opacity.hpp | 2 +- 6 files changed, 62 insertions(+), 34 deletions(-) diff --git a/.gitmodules b/.gitmodules index a69ed97..0fa61a2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,12 @@ [submodule "external/parthenon"] - path = external/parthenon - url = https://github.com/parthenon-hpc-lab/parthenon.git + path = external/parthenon + url = https://github.com/parthenon-hpc-lab/parthenon.git + ignore = dirty [submodule "external/singularity-opac"] - path = external/singularity-opac - url = https://github.com/lanl/singularity-opac.git + path = external/singularity-opac + url = https://github.com/lanl/singularity-opac.git + ignore = dirty [submodule "external/singularity-eos"] - path = external/singularity-eos - url = https://github.com/lanl/singularity-eos.git + path = external/singularity-eos + url = https://github.com/lanl/singularity-eos.git + ignore = dirty diff --git a/external/singularity-opac b/external/singularity-opac index 60b7411..98f27de 160000 --- a/external/singularity-opac +++ b/external/singularity-opac @@ -1 +1 @@ -Subproject commit 60b7411fae78175c26e71e6dfa804bd1b1cc0e1d +Subproject commit 98f27de5364468e90570ed2a4737d7a13f970bde diff --git a/src/jaybenne/jaybenne.cpp b/src/jaybenne/jaybenne.cpp index 9433d67..a831119 100644 --- a/src/jaybenne/jaybenne.cpp +++ b/src/jaybenne/jaybenne.cpp @@ -448,6 +448,7 @@ TaskStatus UpdateDerivedTransportFieldsImpl(MeshData *md, const Real dt) { Real dlnu = -1.0; Real h = -1.0; Real sb = -1.0; + Real ac = -1.0; // radiation constant times light speed (set below) std::vector nu_grid = JaybenneNull>(); ParArray1D nu_bins; @@ -458,6 +459,7 @@ TaskStatus UpdateDerivedTransportFieldsImpl(MeshData *md, const Real dt) { const OpacityAveraging gmode = use_planck ? Planck : Rosseland; if constexpr (FT == FrequencyType::gray) { + ac = 4.0 * (jbn->template Param("stefan_boltzmann")); mopacity = jbn->template Param("mopacity_d"); mscattering = jbn->template Param("mscattering_d"); } else if constexpr (FT == FrequencyType::multigroup) { @@ -498,6 +500,7 @@ TaskStatus UpdateDerivedTransportFieldsImpl(MeshData *md, const Real dt) { const Real temp = eos.TemperatureFromDensityInternalEnergy(rho, sie); const Real cv = eos.SpecificHeatFromDensityInternalEnergy(rho, sie); Real emis = JaybenneNull(); + [[maybe_unused]] const auto acd = ac; [[maybe_unused]] const auto gmoded = gmode; [[maybe_unused]] auto mopac = mopacity; [[maybe_unused]] auto opac = opacity; @@ -505,7 +508,8 @@ TaskStatus UpdateDerivedTransportFieldsImpl(MeshData *md, const Real dt) { [[maybe_unused]] const auto &nu_binsd = nu_bins; [[maybe_unused]] const auto dlnud = dlnu; if constexpr (FT == FrequencyType::gray) { - emis = mopac.Emissivity(rho, temp, gmoded); + const Real T4 = SQR(SQR(temp)); + emis = mopac.AbsorptionCoefficient(rho, temp, 0, gmoded) * acd * T4; } else if constexpr (FT == FrequencyType::multigroup) { // NOTE: 'emis = opac.Emissivity(rho, temp);' may not integrate emis = 0.0; @@ -609,10 +613,10 @@ TaskStatus UpdateDerivedTransportFieldsImpl(MeshData *md, const Real dt) { [[maybe_unused]] const auto tau_ddmcd = tau_ddmc; [[maybe_unused]] const auto lam_extd = lam_ext; if constexpr (FT == FrequencyType::gray) { - ss_l = mscatter.RosselandMeanTotalScatteringCoefficient(rho_l, temp_l); - aa_l = mopac.AbsorptionCoefficient(rho_l, temp_l, gmode2d); - ss_u = mscatter.RosselandMeanTotalScatteringCoefficient(rho_u, temp_u); - aa_u = mopac.AbsorptionCoefficient(rho_u, temp_u, gmode2d); + ss_l = mscatter.ScatteringCoefficient(rho_l, temp_l, 0, gmode2d); + aa_l = mopac.AbsorptionCoefficient(rho_l, temp_l, 0, gmode2d); + ss_u = mscatter.ScatteringCoefficient(rho_u, temp_u, 0, gmode2d); + aa_u = mopac.AbsorptionCoefficient(rho_u, temp_u, 0, gmode2d); // calculate optical thicknesses from lower and upper cell const Real tau_lmin = dx_lmin * (ss_l + aa_l); @@ -725,10 +729,10 @@ TaskStatus UpdateDerivedTransportFieldsImpl(MeshData *md, const Real dt) { [[maybe_unused]] const auto tau_ddmcd = tau_ddmc; [[maybe_unused]] const auto lam_extd = lam_ext; if constexpr (FT == FrequencyType::gray) { - ss_l = mscatter.RosselandMeanTotalScatteringCoefficient(rho_l, temp_l); - aa_l = mopac.AbsorptionCoefficient(rho_l, temp_l, gmode2d); - ss_u = mscatter.RosselandMeanTotalScatteringCoefficient(rho_u, temp_u); - aa_u = mopac.AbsorptionCoefficient(rho_u, temp_u, gmode2d); + ss_l = mscatter.ScatteringCoefficient(rho_l, temp_l, 0, gmode2d); + aa_l = mopac.AbsorptionCoefficient(rho_l, temp_l, 0, gmode2d); + ss_u = mscatter.ScatteringCoefficient(rho_u, temp_u, 0, gmode2d); + aa_u = mopac.AbsorptionCoefficient(rho_u, temp_u, 0, gmode2d); // calculate optical thicknesses from lower and upper cell const Real tau_lmin = dx_lmin * (ss_l + aa_l); @@ -843,10 +847,10 @@ TaskStatus UpdateDerivedTransportFieldsImpl(MeshData *md, const Real dt) { [[maybe_unused]] const auto tau_ddmcd = tau_ddmc; [[maybe_unused]] const auto lam_extd = lam_ext; if constexpr (FT == FrequencyType::gray) { - ss_l = mscatter.RosselandMeanTotalScatteringCoefficient(rho_l, temp_l); - aa_l = mopac.AbsorptionCoefficient(rho_l, temp_l, gmode2d); - ss_u = mscatter.RosselandMeanTotalScatteringCoefficient(rho_u, temp_u); - aa_u = mopac.AbsorptionCoefficient(rho_u, temp_u, gmode2d); + ss_l = mscatter.ScatteringCoefficient(rho_l, temp_l, 0, gmode2d); + aa_l = mopac.AbsorptionCoefficient(rho_l, temp_l, 0, gmode2d); + ss_u = mscatter.ScatteringCoefficient(rho_u, temp_u, 0, gmode2d); + aa_u = mopac.AbsorptionCoefficient(rho_u, temp_u, 0, gmode2d); // calculate optical thicknesses from lower and upper cell const Real tau_lmin = dx_lmin * (ss_l + aa_l); diff --git a/src/jaybenne/sourcing.cpp b/src/jaybenne/sourcing.cpp index 62e331b..dfbd7e5 100644 --- a/src/jaybenne/sourcing.cpp +++ b/src/jaybenne/sourcing.cpp @@ -190,7 +190,9 @@ TaskStatus SourcePhotons(T *md, const Real t_start, const Real dt) { } else if constexpr (ST == SourceType::emission) { Real emis = JaybenneNull(); if constexpr (FT == FrequencyType::gray) { - emis = mopac.Emissivity(rho, temp, gmode); + const Real T4 = SQR(SQR(temp)); + emis = + mopac.AbsorptionCoefficient(rho, temp, 0, gmoded) * 4.0 * sbd * T4; } else if constexpr (FT == FrequencyType::multigroup) { // Construct emission CDF // calculate bin width (assuming log bin width) diff --git a/src/mcblock/mcblock.cpp b/src/mcblock/mcblock.cpp index 8a2e340..c49d850 100644 --- a/src/mcblock/mcblock.cpp +++ b/src/mcblock/mcblock.cpp @@ -112,19 +112,24 @@ std::shared_ptr Initialize(ParameterInput *pin) { MeanOpacity mopacity; std::string abs_model = pin->GetString("mcblock/absorption", "opacity_model"); if (frequency_type == FrequencyType::gray) { + + const std::array gray_bounds = {1.e12, 3.e20}; + if (abs_model == "none") { auto opac = singularity::photons::Gray(1.e-100); mopacity = singularity::photons::MeanNonCGSUnits( - singularity::photons::MeanOpacityBase(opac, -1, 1, 2, -1, 1, 2), time_scale, - mass_scale, length_scale, temperature_scale); + singularity::photons::MeanOpacityBase(opac, -1, 1, 2, -1, 1, 2, gray_bounds, + 1), + time_scale, mass_scale, length_scale, temperature_scale); } else if (abs_model == "constant") { Real kappa = pin->GetReal("mcblock/absorption", "constant_value"); auto opac = singularity::photons::Gray(kappa); mopacity = singularity::photons::MeanNonCGSUnits( - singularity::photons::MeanOpacityBase(opac, -1, 1, 2, -1, 1, 2), time_scale, - mass_scale, length_scale, temperature_scale); + singularity::photons::MeanOpacityBase(opac, -1, 1, 2, -1, 1, 2, gray_bounds, + 1), + time_scale, mass_scale, length_scale, temperature_scale); } else if (abs_model == "table") { std::string table_filename = pin->GetString("mcblock/absorption", "opacity_table"); mopacity = @@ -186,18 +191,21 @@ std::shared_ptr Initialize(ParameterInput *pin) { std::string sct_model = pin->GetOrAddString("mcblock/scattering", "opacity_model", "none"); if (frequency_type == FrequencyType::gray) { + const std::array gray_bounds = {1.e12, 3.e20}; if (sct_model == "none") { auto sopac = singularity::photons::GrayS(0.0, apm); mscattering = - singularity::photons::MeanNonCGSUnitsS( - singularity::photons::MeanSOpacityCGS(sopac, -1., 1., 2, -1., 1., 2), + singularity::photons::MeanNonCGSUnitsS( + singularity::photons::MeanSOpacityBase(sopac, -1., 1., 2, -1., 1., 2, + gray_bounds, 1), time_scale, mass_scale, length_scale, 1.); } else if (sct_model == "constant") { Real kappa_s = pin->GetReal("mcblock/scattering", "constant_value"); auto sopac = singularity::photons::GrayS(kappa_s, apm); mscattering = - singularity::photons::MeanNonCGSUnitsS( - singularity::photons::MeanSOpacityCGS(sopac, -1., 1., 2, -1., 1., 2), + singularity::photons::MeanNonCGSUnitsS( + singularity::photons::MeanSOpacityBase(sopac, -1., 1., 2, -1., 1., 2, + gray_bounds, 1), time_scale, mass_scale, length_scale, 1.); } else { PARTHENON_FAIL("Only none or constant scattering models supported!"); @@ -321,8 +329,8 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { const Real &rho = vmesh(b, fm::density(), k, j, i); const Real &sie = vmesh(b, fm::sie(), k, j, i); const Real temp = eos.TemperatureFromDensityInternalEnergy(rho, sie); - const Real aa = mopacity.AbsorptionCoefficient(rho, temp, gmode); - const Real ss = mscattering.RosselandMeanTotalScatteringCoefficient(rho, temp); + const Real aa = mopacity.AbsorptionCoefficient(rho, temp, 0, gmode); + const Real ss = mscattering.ScatteringCoefficient(rho, temp, 0, gmode); vmesh(b, fm::absorption_opacity(), k, j, i) = aa; vmesh(b, fm::scattering_opacity(), k, j, i) = ss; }); @@ -338,6 +346,10 @@ void UpdateDerived(MeshData *md) { PARTHENON_INSTRUMENT namespace fm = field::material; using parthenon::MakePackDescriptor; + using singularity::photons::OpacityAveraging; + using singularity::photons::Planck; + using singularity::photons::Rosseland; + auto pm = md->GetParentPointer(); auto &resolved_pkgs = pm->resolved_packages; auto &jbn = pm->packages.Get("jaybenne"); @@ -376,6 +388,13 @@ void UpdateDerived(MeshData *md) { // update opacity (TODO: only gray for now) if (frequency_type == FrequencyType::gray) { + + // get opacity average indicators + const auto &use_planck = jbn->template Param("use_planck"); + const auto &use_rosseland = jbn->template Param("use_rosseland"); + // set opacity mode for grey opacity + const OpacityAveraging gmode = (use_planck && !use_rosseland) ? Planck : Rosseland; + parthenon::par_for( DEFAULT_LOOP_PATTERN, "Update opacity", parthenon::DevExecSpace(), 0, vmesh.GetNBlocks() - 1, kbe.s, kbe.e, jbe.s, jbe.e, ibe.s, ibe.e, @@ -383,8 +402,8 @@ void UpdateDerived(MeshData *md) { const Real &rho = vmesh(b, fm::density(), k, j, i); const Real &sie = vmesh(b, fm::sie(), k, j, i); const Real temp = eos.TemperatureFromDensityInternalEnergy(rho, sie); - const Real aa = mopacity.AbsorptionCoefficient(rho, temp); - const Real ss = mscattering.RosselandMeanTotalScatteringCoefficient(rho, temp); + const Real aa = mopacity.AbsorptionCoefficient(rho, temp, 0, gmode); + const Real ss = mscattering.ScatteringCoefficient(rho, temp, 0, gmode); vmesh(b, fm::absorption_opacity(), k, j, i) = aa; vmesh(b, fm::scattering_opacity(), k, j, i) = ss; }); diff --git a/src/mcblock/opacity.hpp b/src/mcblock/opacity.hpp index 4ddbb40..734d879 100644 --- a/src/mcblock/opacity.hpp +++ b/src/mcblock/opacity.hpp @@ -37,7 +37,7 @@ using Scattering = singularity::photons::impl::S_Variant< singularity::photons::NonCGSUnitsS>; using MeanScattering = - singularity::photons::MeanNonCGSUnitsS; + singularity::photons::MeanNonCGSUnitsS; } // namespace mcblock From b5119b652b808724c5d25e9dfe86036b64c9ace6 Mon Sep 17 00:00:00 2001 From: Ryan Thomas Wollaeger Date: Mon, 3 Aug 2026 11:04:08 -0600 Subject: [PATCH 2/5] Bump singularity-opac. --- external/singularity-opac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/singularity-opac b/external/singularity-opac index 98f27de..b97515f 160000 --- a/external/singularity-opac +++ b/external/singularity-opac @@ -1 +1 @@ -Subproject commit 98f27de5364468e90570ed2a4737d7a13f970bde +Subproject commit b97515f695f96fabeca28852b5483bc0cd49528e From 343546db2b96d31b88a97a6cd9df69583c41785a Mon Sep 17 00:00:00 2001 From: Ryan Thomas Wollaeger Date: Mon, 3 Aug 2026 12:37:28 -0600 Subject: [PATCH 3/5] Bump singularity-eos. --- env/bash | 4 ++-- external/singularity-eos | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/env/bash b/env/bash index b932cc0..3ab9707 100644 --- a/env/bash +++ b/env/bash @@ -108,8 +108,8 @@ elif [[ $PARTITION == "darwin-volta-x86" ]]; then module load cuda/12.3.1 module load gcc/12.2.0 module load hdf5/1.12.2 - module load miniconda3 - source /projects/opt/centos8/x86_64/miniconda3/py39_4.12.0/bin/activate /usr/projects/jovian/dependencies/python/volta-x86-py39 + module load miniconda3/py311_23.11.0 + source /usr/projects/jovian/dependencies/python/volta-x86-py311/bin/activate shorten_prompt module load cmake/3.26.3 module list diff --git a/external/singularity-eos b/external/singularity-eos index 4093f31..e32a25b 160000 --- a/external/singularity-eos +++ b/external/singularity-eos @@ -1 +1 @@ -Subproject commit 4093f31864d49f3688884e397481d0ecd89f87e9 +Subproject commit e32a25bed7b73baa7a5684c0183d2c369e16693c From 69b27fdf2d02448ee844df51bc60a292c7e3d57a Mon Sep 17 00:00:00 2001 From: Ryan Thomas Wollaeger Date: Mon, 3 Aug 2026 13:26:30 -0600 Subject: [PATCH 4/5] Try claude-opus suggestion to avoid UCX use of gdr_copy. --- env/bash | 1 + 1 file changed, 1 insertion(+) diff --git a/env/bash b/env/bash index 3ab9707..e9fc9d5 100644 --- a/env/bash +++ b/env/bash @@ -113,6 +113,7 @@ elif [[ $PARTITION == "darwin-volta-x86" ]]; then shorten_prompt module load cmake/3.26.3 module list + export UCX_TLS=^gdr_copy # claude-opus suggestion for GPU buffer-pinning error (may just be node cn410) echo "...setup SUCCEEDED" elif [[ $PARTITION == "darwin-a100" ]]; then module purge From cc83bf1d8bd4728cd2444eac1fc8b4fb95b773f1 Mon Sep 17 00:00:00 2001 From: Ryan Thomas Wollaeger Date: Tue, 4 Aug 2026 10:58:41 -0600 Subject: [PATCH 5/5] Scale hard-coded gray values (not needed if/when set to 0 to inf). --- src/mcblock/mcblock.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mcblock/mcblock.cpp b/src/mcblock/mcblock.cpp index c49d850..f386e84 100644 --- a/src/mcblock/mcblock.cpp +++ b/src/mcblock/mcblock.cpp @@ -113,7 +113,8 @@ std::shared_ptr Initialize(ParameterInput *pin) { std::string abs_model = pin->GetString("mcblock/absorption", "opacity_model"); if (frequency_type == FrequencyType::gray) { - const std::array gray_bounds = {1.e12, 3.e20}; + // hard-coded numbers in Hz + const std::array gray_bounds = {time_scale * 1.e12, time_scale * 3.e20}; if (abs_model == "none") { auto opac = singularity::photons::Gray(1.e-100);