From 0d4fd606dc78faf99b743c7e2f26581f38820367 Mon Sep 17 00:00:00 2001 From: AlexBogaev Date: Tue, 9 Jun 2026 02:41:32 -0400 Subject: [PATCH 1/8] Refactor doInSituViz to use constructPlotMF() for dynamic plot variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracts constructPlotMF() from WritePlotFile() following PeleC's pattern. Both WritePlotFile() and doInSituViz() now call the same helper, ensuring Ascent publishes the same fields as the plotfile including: - Full state respecting m_plotStateSpec - Plasma fields (PELE_USE_PLASMA: nE, phiV, ion drift fluxes) - Soot moments (PELE_USE_SOOT) - Radiation fields (PELE_USE_RADIATION) - Reaction rates, FunctCall, HeatRelease - Derived variables (amr.derive_plot_vars) - Auxiliary variables, LES viscosity, ODE quantities - Spray variables (PELE_USE_SPRAY) - EB volume fraction (AMREX_USE_EB) Also adds AMREX_ASSERT(ncomp == a_plt_VarsName.size()) mirroring PeleC's constructPlotMF sanity check, and fixes doInSituViz to call removeVirtualParticles() after constructPlotMF when PELE_USE_SPRAY is active, matching the cleanup WritePlotFile performs after SprayParticleIO. Tested on cbogaev (Ubuntu 24.04, sm_89, OpenMPI 5.0.5, CUDA 12.6): Stage 1 — HotBubble/air, 4 AMR levels, 2D: temp, avg_pressure, mag_vort, Y(N2) from mass_fractions derive, m_plotStateSpec=0 suppression verified, LES viscturb (Smagorinsky) Stage 2 — FlameSheet/drm19, 3 AMR levels, 2D: temp, I_R(CH4), FunctCall, HeatRelease, mixture_fraction, progress_variable Spray (PELE_USE_SPRAY), plasma, soot, radiation deferred — require compile-time flags not present in test cases. --- Source/PeleLMeX.H | 3 + Source/PeleLMeX_InSituViz.cpp | 57 +++------ Source/PeleLMeX_Plot.cpp | 231 ++++++++++++++++++---------------- 3 files changed, 148 insertions(+), 143 deletions(-) diff --git a/Source/PeleLMeX.H b/Source/PeleLMeX.H index dad8bb0f5..56b21e4e2 100644 --- a/Source/PeleLMeX.H +++ b/Source/PeleLMeX.H @@ -1019,6 +1019,9 @@ public: //----------------------------------------------------------------------------- // I/O void WritePlotFile(); + void constructPlotMF( + amrex::Vector& a_mf_plt, + amrex::Vector& a_plt_VarsName); bool writePlotNow() const; bool checkMessage(const std::string& a_action) const; void WriteCheckPointFile(); diff --git a/Source/PeleLMeX_InSituViz.cpp b/Source/PeleLMeX_InSituViz.cpp index c4345110e..8c716b066 100644 --- a/Source/PeleLMeX_InSituViz.cpp +++ b/Source/PeleLMeX_InSituViz.cpp @@ -19,52 +19,33 @@ PeleLM::doInSituViz() auto dPlotFileTime0 = amrex::second(); - averageDownState(AmrNewTime); - - amrex::Vector speciesNames; - pele::physics::eos::speciesNames( - speciesNames, &(eos_parms.host_parm())); - - amrex::Vector plt_var_names; - AMREX_D_TERM(plt_var_names.push_back("x_velocity"); - , plt_var_names.push_back("y_velocity"); - , plt_var_names.push_back("z_velocity")); - plt_var_names.push_back("density"); - for (int n = 0; n < NUM_SPECIES; ++n) { - plt_var_names.push_back("rho.Y(" + speciesNames[n] + ")"); - } - plt_var_names.push_back("rhoh"); - plt_var_names.push_back("temp"); - plt_var_names.push_back("RhoRT"); - - // Base state: velocity + density + species + rhoh + temp + RhoRT - const int ncomp = RHORT + 1; - AMREX_ASSERT(ncomp == static_cast(plt_var_names.size())); - - const int nlevels = finest_level + 1; + // Assemble the same plot MultiFab and variable list used by WritePlotFile(). + // This ensures Ascent publishes all user-requested fields (species flag, + // plasma, soot, radiation, reactions, derived vars, spray, LES viscosity, + // ODE quantities, EB volume fraction, external sources) rather than only + // the hardcoded base state. averageDown is performed inside constructPlotMF. + // constructPlotMF also calls setupVirtualParticles(0) when PELE_USE_SPRAY is + // active; we are responsible for the matching removeVirtualParticles cleanup + // since there is no SprayParticleIO step here (unlike WritePlotFile). amrex::Vector plotMFs; - plotMFs.reserve(nlevels); - for (int lev = 0; lev <= finest_level; ++lev) { - plotMFs.emplace_back( - grids[lev], dmap[lev], ncomp, 0, amrex::MFInfo(), Factory(lev)); - amrex::MultiFab::Copy( - plotMFs[lev], m_leveldata_new[lev]->state, 0, 0, ncomp, 0); - } + amrex::Vector plt_var_names; + constructPlotMF(plotMFs, plt_var_names); - amrex::Vector plotMFs_constvec; - plotMFs_constvec.reserve(nlevels); - for (int lev = 0; lev < nlevels; ++lev) { - plotMFs_constvec.push_back( - static_cast(&plotMFs[lev])); +#ifdef PELE_USE_SPRAY + if (do_spray_particles) { + for (int lev = 0; lev <= finest_level; ++lev) { + removeVirtualParticles(lev); + } } +#endif // PeleLMeX is non-subcycling: all levels share m_nstep - amrex::Vector istep(nlevels, m_nstep); + amrex::Vector istep(finest_level + 1, m_nstep); conduit::Node bp_mesh; amrex::MultiLevelToBlueprint( - nlevels, plotMFs_constvec, plt_var_names, Geom(), m_cur_time, istep, - refRatio(), bp_mesh); + finest_level + 1, GetVecOfConstPtrs(plotMFs), plt_var_names, Geom(), + m_cur_time, istep, refRatio(), bp_mesh); ascent::Ascent ascent; conduit::Node open_opts; diff --git a/Source/PeleLMeX_Plot.cpp b/Source/PeleLMeX_Plot.cpp index 7d4c72b58..f8129ce14 100644 --- a/Source/PeleLMeX_Plot.cpp +++ b/Source/PeleLMeX_Plot.cpp @@ -83,13 +83,55 @@ PeleLM::WritePlotFile() amrex::VisMF::SetNOutFiles(m_nfiles); + //---------------------------------------------------------------- + // Average down the state + + amrex::Vector mf_plt; + amrex::Vector plt_VarsName; + constructPlotMF(mf_plt, plt_VarsName); + + // No SubCycling, all levels the same step. + amrex::Vector istep(finest_level + 1, m_nstep); + +#ifdef AMREX_USE_HDF5 + if (m_write_hdf5_pltfile) { + amrex::WriteMultiLevelPlotfileHDF5( + plotfilename, finest_level + 1, GetVecOfConstPtrs(mf_plt), plt_VarsName, + Geom(), m_cur_time, istep, refRatio()); + } else +#endif + { + amrex::WriteMultiLevelPlotfile( + plotfilename, finest_level + 1, GetVecOfConstPtrs(mf_plt), plt_VarsName, + Geom(), m_cur_time, istep, refRatio()); + } + +#ifdef PELE_USE_SPRAY + if (do_spray_particles) { + constexpr bool is_spraycheck = false; + for (int lev = 0; lev <= finest_level; ++lev) { + SprayPC->SprayParticleIO(lev, is_spraycheck, plotfilename); + // Remove virtual particles that were made for derived variables + removeVirtualParticles(lev); + } + } +#endif +} + +void +PeleLM::constructPlotMF( + amrex::Vector& a_mf_plt, + amrex::Vector& a_plt_VarsName) +{ + BL_PROFILE("PeleLMeX::constructPlotMF()"); + //---------------------------------------------------------------- // Average down the state averageDownState(AmrNewTime); if (m_nAux > 0) { averageDownAux(AmrNewTime); } - // Get consistent reaction data across level + // Get consistent reaction data across levels if ((m_do_react != 0) && (m_skipInstantRR == 0) && (m_plot_react != 0)) { averageDownReaction(); } @@ -176,88 +218,88 @@ PeleLM::WritePlotFile() } //---------------------------------------------------------------- - // Plot MultiFabs - amrex::Vector mf_plt; - mf_plt.reserve(finest_level + 1); + // Allocate plot MultiFabs + a_mf_plt.clear(); + a_mf_plt.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - mf_plt.emplace_back( + a_mf_plt.emplace_back( grids[lev], dmap[lev], ncomp, 0, amrex::MFInfo(), Factory(lev)); } //---------------------------------------------------------------- - // Components names + // Component names amrex::Vector names; pele::physics::eos::speciesNames( names, &(eos_parms.host_parm())); - amrex::Vector plt_VarsName; - AMREX_D_TERM(plt_VarsName.push_back("x_velocity"); - , plt_VarsName.push_back("y_velocity"); - , plt_VarsName.push_back("z_velocity")); + a_plt_VarsName.clear(); + AMREX_D_TERM(a_plt_VarsName.push_back("x_velocity"); + , a_plt_VarsName.push_back("y_velocity"); + , a_plt_VarsName.push_back("z_velocity")); if (m_incompressible == 0) { - plt_VarsName.push_back("density"); + a_plt_VarsName.push_back("density"); if (m_plotStateSpec != 0) { for (int n = 0; n < NUM_SPECIES; ++n) { - plt_VarsName.push_back("rho.Y(" + names[n] + ")"); + a_plt_VarsName.push_back("rho.Y(" + names[n] + ")"); } } - plt_VarsName.push_back("rhoh"); - plt_VarsName.push_back("temp"); - plt_VarsName.push_back("RhoRT"); + a_plt_VarsName.push_back("rhoh"); + a_plt_VarsName.push_back("temp"); + a_plt_VarsName.push_back("RhoRT"); #ifdef PELE_USE_PLASMA - plt_VarsName.push_back("nE"); - plt_VarsName.push_back("phiV"); + a_plt_VarsName.push_back("nE"); + a_plt_VarsName.push_back("phiV"); #endif #ifdef PELE_USE_SOOT for (int mom = 0; mom < NUMSOOTVAR; ++mom) { const std::string sootname = soot_model->sootVariableName(mom); - plt_VarsName.push_back(sootname); + a_plt_VarsName.push_back(sootname); } #endif #ifdef PELE_USE_RADIATION if (do_rad_solve) { - plt_VarsName.push_back("rad.G"); - plt_VarsName.push_back("rad.kappa"); - plt_VarsName.push_back("rad.emis"); + a_plt_VarsName.push_back("rad.G"); + a_plt_VarsName.push_back("rad.kappa"); + a_plt_VarsName.push_back("rad.emis"); } #endif if (m_has_divu != 0) { - plt_VarsName.push_back("divu"); + a_plt_VarsName.push_back("divu"); } } if (m_plot_grad_p != 0) { - AMREX_D_TERM(plt_VarsName.push_back("gradpx"); - , plt_VarsName.push_back("gradpy"); - , plt_VarsName.push_back("gradpz")); + AMREX_D_TERM(a_plt_VarsName.push_back("gradpx"); + , a_plt_VarsName.push_back("gradpy"); + , a_plt_VarsName.push_back("gradpz")); } for (int n = 0; n < m_nAux; ++n) { - plt_VarsName.push_back(m_aux_names[n]); + a_plt_VarsName.push_back(m_aux_names[n]); } if ((m_do_react != 0) && (m_skipInstantRR == 0) && (m_plot_react != 0)) { for (int n = 0; n < NUM_SPECIES; ++n) { - plt_VarsName.push_back("I_R(" + names[n] + ")"); + a_plt_VarsName.push_back("I_R(" + names[n] + ")"); } #ifdef PELE_USE_PLASMA - plt_VarsName.push_back("I_R(nE)"); + a_plt_VarsName.push_back("I_R(nE)"); #endif - plt_VarsName.push_back("FunctCall"); + a_plt_VarsName.push_back("FunctCall"); // Extras: if (m_plotHeatRelease != 0) { - plt_VarsName.push_back("HeatRelease"); + a_plt_VarsName.push_back("HeatRelease"); } } #ifdef AMREX_USE_EB - plt_VarsName.push_back("volFrac"); + a_plt_VarsName.push_back("volFrac"); #endif for (int ivar = 0; ivar < m_derivePlotVarCount; ++ivar) { const PeleLMDeriveRec* rec = derive_lst.get(m_derivePlotVars[ivar]); for (int dvar = 0; dvar < rec->numDerive(); ++dvar) { - plt_VarsName.push_back(rec->variableName(dvar)); + a_plt_VarsName.push_back(rec->variableName(dvar)); } } #ifdef PELE_USE_SPRAY @@ -266,18 +308,18 @@ PeleLM::WritePlotFile() setupVirtualParticles(0); for (const auto& spray_derive_name : SprayParticleContainer::DeriveVarNames()) { - plt_VarsName.push_back(spray_derive_name); + a_plt_VarsName.push_back(spray_derive_name); } } if (do_spray_particles && SprayParticleContainer::plot_spray_src) { - plt_VarsName.push_back("spray_mass_src"); - plt_VarsName.push_back("spray_energy_src"); - AMREX_D_TERM(plt_VarsName.push_back("spray_momentumX_src"); - , plt_VarsName.push_back("spray_momentumY_src"); - , plt_VarsName.push_back("spray_momentumZ_src")); + a_plt_VarsName.push_back("spray_mass_src"); + a_plt_VarsName.push_back("spray_energy_src"); + AMREX_D_TERM(a_plt_VarsName.push_back("spray_momentumX_src"); + , a_plt_VarsName.push_back("spray_momentumY_src"); + , a_plt_VarsName.push_back("spray_momentumZ_src")); for (const auto& spray_fuel_name : SprayParticleContainer::m_sprayDepNames) { - plt_VarsName.push_back("spray_" + spray_fuel_name + "_src"); + a_plt_VarsName.push_back("spray_" + spray_fuel_name + "_src"); } } #endif @@ -287,7 +329,7 @@ PeleLM::WritePlotFile() for (int ivar = 0; ivar < NUM_IONS; ++ivar) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const std::string dir = (idim == 0) ? "X" : ((idim == 1) ? "Y" : "Z"); - plt_VarsName.push_back( + a_plt_VarsName.push_back( "DriftFlux_" + names[NUM_SPECIES - NUM_IONS + ivar] + "_" + dir); } } @@ -295,19 +337,19 @@ PeleLM::WritePlotFile() #endif if (m_do_les && m_plot_les) { - plt_VarsName.push_back("viscturb"); + a_plt_VarsName.push_back("viscturb"); } #if NUM_ODE > 0 for (int n = 0; n < NUM_ODE; ++n) { - plt_VarsName.push_back(m_ode_names[n]); + a_plt_VarsName.push_back(m_ode_names[n]); } #endif // External source terms if (m_plot_extSource) { for (int ivar = 0; ivar < NVAR; ++ivar) { - plt_VarsName.push_back("extsource_" + stateVariableName(ivar)); + a_plt_VarsName.push_back("extsource_" + stateVariableName(ivar)); } } @@ -317,131 +359,134 @@ PeleLM::WritePlotFile() int cnt = 0; if (m_incompressible != 0) { amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, 0, cnt, AMREX_SPACEDIM, 0); + a_mf_plt[lev], m_leveldata_new[lev]->state, 0, cnt, AMREX_SPACEDIM, 0); cnt += AMREX_SPACEDIM; } else { // Velocity and density amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, 0, cnt, AMREX_SPACEDIM + 1, + a_mf_plt[lev], m_leveldata_new[lev]->state, 0, cnt, AMREX_SPACEDIM + 1, 0); cnt += AMREX_SPACEDIM + 1; // Species only if requested if (m_plotStateSpec != 0) { amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, FIRSTSPEC, cnt, NUM_SPECIES, - 0); + a_mf_plt[lev], m_leveldata_new[lev]->state, FIRSTSPEC, cnt, + NUM_SPECIES, 0); cnt += NUM_SPECIES; } amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, RHOH, cnt, 3, 0); + a_mf_plt[lev], m_leveldata_new[lev]->state, RHOH, cnt, 3, 0); cnt += 3; #ifdef PELE_USE_PLASMA amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, NE, cnt, 2, 0); + a_mf_plt[lev], m_leveldata_new[lev]->state, NE, cnt, 2, 0); cnt += 2; #endif #ifdef PELE_USE_SOOT amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, FIRSTSOOT, cnt, NUMSOOTVAR, - 0); + a_mf_plt[lev], m_leveldata_new[lev]->state, FIRSTSOOT, cnt, + NUMSOOTVAR, 0); cnt += NUMSOOTVAR; #endif #ifdef PELE_USE_RADIATION if (do_rad_solve) { - amrex::MultiFab::Copy(mf_plt[lev], rad_model->G()[lev], 0, cnt, 1, 0); + amrex::MultiFab::Copy( + a_mf_plt[lev], rad_model->G()[lev], 0, cnt, 1, 0); cnt += 1; amrex::MultiFab::Copy( - mf_plt[lev], rad_model->kappa()[lev], 0, cnt, 1, 0); + a_mf_plt[lev], rad_model->kappa()[lev], 0, cnt, 1, 0); cnt += 1; amrex::MultiFab::Copy( - mf_plt[lev], rad_model->emis()[lev], 0, cnt, 1, 0); + a_mf_plt[lev], rad_model->emis()[lev], 0, cnt, 1, 0); cnt += 1; } #endif if (m_has_divu != 0) { amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->divu, 0, cnt, 1, 0); + a_mf_plt[lev], m_leveldata_new[lev]->divu, 0, cnt, 1, 0); cnt += 1; } } if (m_plot_grad_p != 0) { amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->gp, 0, cnt, AMREX_SPACEDIM, 0); + a_mf_plt[lev], m_leveldata_new[lev]->gp, 0, cnt, AMREX_SPACEDIM, 0); cnt += AMREX_SPACEDIM; } if (m_nAux > 0) { amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->auxiliaries, 0, cnt, m_nAux, 0); + a_mf_plt[lev], m_leveldata_new[lev]->auxiliaries, 0, cnt, m_nAux, 0); cnt += m_nAux; } if ((m_do_react != 0) && (m_skipInstantRR == 0) && (m_plot_react != 0)) { amrex::MultiFab::Copy( - mf_plt[lev], m_leveldatareact[lev]->I_R, 0, cnt, nCompIR(), 0); + a_mf_plt[lev], m_leveldatareact[lev]->I_R, 0, cnt, nCompIR(), 0); cnt += nCompIR(); amrex::MultiFab::Copy( - mf_plt[lev], m_leveldatareact[lev]->functC, 0, cnt, 1, 0); + a_mf_plt[lev], m_leveldatareact[lev]->functC, 0, cnt, 1, 0); cnt += 1; if (m_plotHeatRelease != 0) { std::unique_ptr mf; mf = std::make_unique(grids[lev], dmap[lev], 1, 0); getHeatRelease(lev, mf.get()); - amrex::MultiFab::Copy(mf_plt[lev], *mf, 0, cnt, 1, 0); + amrex::MultiFab::Copy(a_mf_plt[lev], *mf, 0, cnt, 1, 0); cnt += 1; } } #ifdef AMREX_USE_EB amrex::MultiFab::Copy( - mf_plt[lev], EBFactory(lev).getVolFrac(), 0, cnt, 1, 0); + a_mf_plt[lev], EBFactory(lev).getVolFrac(), 0, cnt, 1, 0); cnt += 1; #endif for (int ivar = 0; ivar < m_derivePlotVarCount; ++ivar) { std::unique_ptr mf; mf = derive(m_derivePlotVars[ivar], m_cur_time, lev, 0); - amrex::MultiFab::Copy(mf_plt[lev], *mf, 0, cnt, mf->nComp(), 0); + amrex::MultiFab::Copy(a_mf_plt[lev], *mf, 0, cnt, mf->nComp(), 0); cnt += mf->nComp(); } #ifdef PELE_USE_SPRAY if (SprayParticleContainer::NumDeriveVars() > 0) { const int num_spray_derive = SprayParticleContainer::NumDeriveVars(); - mf_plt[lev].setVal(0., cnt, num_spray_derive); - SprayPC->computeDerivedVars(mf_plt[lev], lev, cnt); + a_mf_plt[lev].setVal(0., cnt, num_spray_derive); + SprayPC->computeDerivedVars(a_mf_plt[lev], lev, cnt); if (lev < finest_level) { amrex::MultiFab tmp_plt( grids[lev], dmap[lev], num_spray_derive, 0, amrex::MFInfo(), Factory(lev)); tmp_plt.setVal(0.); VirtPC->computeDerivedVars(tmp_plt, lev, 0); - amrex::MultiFab::Add(mf_plt[lev], tmp_plt, 0, cnt, num_spray_derive, 0); + amrex::MultiFab::Add( + a_mf_plt[lev], tmp_plt, 0, cnt, num_spray_derive, 0); } cnt += num_spray_derive; } if (do_spray_particles && SprayParticleContainer::plot_spray_src) { SprayComps scomps = SprayParticleContainer::getSprayComps(); amrex::MultiFab::Copy( - mf_plt[lev], *m_spraysource[lev], scomps.rhoSrcIndx, cnt++, 1, 0); + a_mf_plt[lev], *m_spraysource[lev], scomps.rhoSrcIndx, cnt++, 1, 0); amrex::MultiFab::Copy( - mf_plt[lev], *m_spraysource[lev], scomps.engSrcIndx, cnt++, 1, 0); + a_mf_plt[lev], *m_spraysource[lev], scomps.engSrcIndx, cnt++, 1, 0); amrex::MultiFab::Copy( - mf_plt[lev], *m_spraysource[lev], scomps.momSrcIndx, cnt, + a_mf_plt[lev], *m_spraysource[lev], scomps.momSrcIndx, cnt, AMREX_SPACEDIM, 0); cnt += AMREX_SPACEDIM; for (int spf = 0; spf < SPRAY_FUEL_NUM; ++spf) { amrex::MultiFab::Copy( - mf_plt[lev], *m_spraysource[lev], scomps.specSrcIndx + spf, cnt++, 1, - 0); + a_mf_plt[lev], *m_spraysource[lev], scomps.specSrcIndx + spf, + cnt++, 1, 0); } } #endif #ifdef PELE_USE_PLASMA if (m_do_extraEFdiags) { amrex::MultiFab::Copy( - mf_plt[lev], *m_ionsFluxes[lev], 0, cnt, m_ionsFluxes[lev]->nComp(), 0); + a_mf_plt[lev], *m_ionsFluxes[lev], 0, cnt, + m_ionsFluxes[lev]->nComp(), 0); cnt += m_ionsFluxes[lev]->nComp(); } #endif @@ -449,7 +494,7 @@ PeleLM::WritePlotFile() if (m_do_les && m_plot_les) { constexpr amrex::Real fact = 0.5 / static_cast(AMREX_SPACEDIM); - auto const& plot_arr = mf_plt[lev].arrays(); + auto const& plot_arr = a_mf_plt[lev].arrays(); AMREX_D_TERM( auto const& mut_arr_x = m_leveldata_new[lev]->visc_turb_fc[0].const_arrays(); @@ -459,10 +504,10 @@ PeleLM::WritePlotFile() m_leveldata_new[lev]->visc_turb_fc[2].const_arrays();) // interpolate turbulent viscosity from faces to centers amrex::ParallelFor( - mf_plt[lev], [plot_arr, cnt, mut_arr_x, mut_arr_y + a_mf_plt[lev], [plot_arr, cnt, mut_arr_x, mut_arr_y #if (AMREX_SPACEDIM == 3) - , - mut_arr_z + , + mut_arr_z #endif ] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { plot_arr[box_no](i, j, k, cnt) = @@ -478,49 +523,25 @@ PeleLM::WritePlotFile() #if NUM_ODE > 0 amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, FIRSTODE, cnt, NUM_ODE, 0); + a_mf_plt[lev], m_leveldata_new[lev]->state, FIRSTODE, cnt, NUM_ODE, 0); cnt += NUM_ODE; #endif if (m_plot_extSource) { - amrex::MultiFab::Copy(mf_plt[lev], *m_extSource[lev], 0, cnt, NVAR, 0); + amrex::MultiFab::Copy(a_mf_plt[lev], *m_extSource[lev], 0, cnt, NVAR, 0); } #ifdef AMREX_USE_EB if (m_plot_zeroEBcovered != 0) { - EB_set_covered(mf_plt[lev], 0.0); + EB_set_covered(a_mf_plt[lev], 0.0); } #endif } - // No SubCycling, all levels the same step. - amrex::Vector istep(finest_level + 1, m_nstep); - -#ifdef AMREX_USE_HDF5 - if (m_write_hdf5_pltfile) { - amrex::WriteMultiLevelPlotfileHDF5( - plotfilename, finest_level + 1, GetVecOfConstPtrs(mf_plt), plt_VarsName, - Geom(), m_cur_time, istep, refRatio()); - } else -#endif - { - amrex::WriteMultiLevelPlotfile( - plotfilename, finest_level + 1, GetVecOfConstPtrs(mf_plt), plt_VarsName, - Geom(), m_cur_time, istep, refRatio()); - } - -#ifdef PELE_USE_SPRAY - if (do_spray_particles) { - constexpr bool is_spraycheck = false; - for (int lev = 0; lev <= finest_level; ++lev) { - SprayPC->SprayParticleIO(lev, is_spraycheck, plotfilename); - // Remove virtual particles that were made for derived variables - removeVirtualParticles(lev); - } - } -#endif + // Sanity check: ncomp and name list must stay in sync. + // Mirrors PeleC's constructPlotMF AMREX_ASSERT(n_data_items == plt_var_names.size()). + AMREX_ASSERT(ncomp == static_cast(a_plt_VarsName.size())); } - void PeleLM::WriteHeader(const std::string& name, const bool is_checkpoint) const { From a83d4a4ce53fdc92a468f763160738a7e8477fc7 Mon Sep 17 00:00:00 2001 From: AlexBogaev Date: Wed, 10 Jun 2026 01:33:48 -0400 Subject: [PATCH 2/8] Apply clang-format corrections to PeleLMeX_Plot.cpp Fix line wrapping in constructPlotMF() to satisfy clang-format20: - PELE_USE_SOOT MultiFab::Copy FIRSTSOOT args - PELE_USE_RADIATION rad_model->G() single-line collapse - PELE_USE_SPRAY specSrcIndx args - PELE_USE_PLASMA ionsFluxes args - AMREX_ASSERT comment reflow --- Source/PeleLMeX_Plot.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Source/PeleLMeX_Plot.cpp b/Source/PeleLMeX_Plot.cpp index f8129ce14..6273ce81c 100644 --- a/Source/PeleLMeX_Plot.cpp +++ b/Source/PeleLMeX_Plot.cpp @@ -384,14 +384,13 @@ PeleLM::constructPlotMF( #endif #ifdef PELE_USE_SOOT amrex::MultiFab::Copy( - a_mf_plt[lev], m_leveldata_new[lev]->state, FIRSTSOOT, cnt, - NUMSOOTVAR, 0); + a_mf_plt[lev], m_leveldata_new[lev]->state, FIRSTSOOT, cnt, NUMSOOTVAR, + 0); cnt += NUMSOOTVAR; #endif #ifdef PELE_USE_RADIATION if (do_rad_solve) { - amrex::MultiFab::Copy( - a_mf_plt[lev], rad_model->G()[lev], 0, cnt, 1, 0); + amrex::MultiFab::Copy(a_mf_plt[lev], rad_model->G()[lev], 0, cnt, 1, 0); cnt += 1; amrex::MultiFab::Copy( a_mf_plt[lev], rad_model->kappa()[lev], 0, cnt, 1, 0); @@ -477,16 +476,16 @@ PeleLM::constructPlotMF( cnt += AMREX_SPACEDIM; for (int spf = 0; spf < SPRAY_FUEL_NUM; ++spf) { amrex::MultiFab::Copy( - a_mf_plt[lev], *m_spraysource[lev], scomps.specSrcIndx + spf, - cnt++, 1, 0); + a_mf_plt[lev], *m_spraysource[lev], scomps.specSrcIndx + spf, cnt++, + 1, 0); } } #endif #ifdef PELE_USE_PLASMA if (m_do_extraEFdiags) { amrex::MultiFab::Copy( - a_mf_plt[lev], *m_ionsFluxes[lev], 0, cnt, - m_ionsFluxes[lev]->nComp(), 0); + a_mf_plt[lev], *m_ionsFluxes[lev], 0, cnt, m_ionsFluxes[lev]->nComp(), + 0); cnt += m_ionsFluxes[lev]->nComp(); } #endif @@ -539,7 +538,8 @@ PeleLM::constructPlotMF( } // Sanity check: ncomp and name list must stay in sync. - // Mirrors PeleC's constructPlotMF AMREX_ASSERT(n_data_items == plt_var_names.size()). + // Mirrors PeleC's constructPlotMF AMREX_ASSERT(n_data_items == + // plt_var_names.size()). AMREX_ASSERT(ncomp == static_cast(a_plt_VarsName.size())); } void From 8a37d62020875f79993a594efe2dc8e53accbed4 Mon Sep 17 00:00:00 2001 From: AlexBogaev Date: Wed, 10 Jun 2026 19:06:50 -0400 Subject: [PATCH 3/8] Add dedicated InSituViz.rst documentation page Move Ascent in-situ documentation from Tutorials_HotBubble.rst into a dedicated InSituViz.rst page under the Usage section of the manual. InSituViz.rst covers: - Full stack build instructions (MPI, Ascent+Conduit, PeleLMeX) - ascent_options.yaml backend selection and default priority - Complete published fields reference by physics module: base state, reactions, derived vars, LES, soot, radiation, spray, plasma, EB - Runtime field name discovery via intentional Ascent error - Example ascent_actions.yaml files Tutorials_HotBubble.rst retains a short stub with the run example, figure, and a cross-reference link to the new page. --- Docs/sphinx/manual/InSituViz.rst | 550 +++++++++++++++++++++ Docs/sphinx/manual/Tutorials_HotBubble.rst | 121 +---- Docs/sphinx/manual/index.rst | 1 + 3 files changed, 562 insertions(+), 110 deletions(-) create mode 100644 Docs/sphinx/manual/InSituViz.rst diff --git a/Docs/sphinx/manual/InSituViz.rst b/Docs/sphinx/manual/InSituViz.rst new file mode 100644 index 000000000..0fac3f761 --- /dev/null +++ b/Docs/sphinx/manual/InSituViz.rst @@ -0,0 +1,550 @@ +.. role:: cpp(code) + :language: c++ + +.. _sec:insitu: + +In-Situ Visualization with Ascent +================================== + +`PeleLMeX` supports in-situ visualization via `Ascent +`_, an open-source many-core capable +lightweight in-situ visualization and analysis library developed as part of the +`Alpine `_ project. Ascent uses `Conduit +`_ to describe and pass simulation data, +and the `Viskores `_ library for +rendering on both CPU and GPU. Since the solver state is passed directly to +Ascent without writing to disk, in-situ rendering eliminates the I/O bottleneck +of traditional post-hoc workflows and is well-suited to large-scale GPU runs. + +The PeleLMeX Ascent integration publishes exactly the same fields as +``WritePlotFile()``, controlled at runtime by the same input file flags. +Any field visible in a plotfile is also available for in-situ rendering. + +.. _sec:insitu::build: + +Building the full stack +----------------------- + +Ascent in-situ visualization requires that Ascent, Conduit, and PeleLMeX are +all built against the same MPI installation and, for GPU rendering, the same +CUDA toolkit. Building any component against a different MPI will cause ABI +mismatches at runtime. The recommended approach is to build the entire stack in +order: MPI first, then Ascent+Conduit via ``build_ascent.sh``, then PeleLMeX. + +Step 1 — MPI +^^^^^^^^^^^^ + +Build or install an MPI implementation. OpenMPI, MPICH, MVAPICH, Intel MPI, and +Cray MPI are all supported; Ascent uses the standard MPI-2 API and is not tied +to any specific implementation. Record the install prefix — it is needed for +every subsequent step. :: + + # Example: OpenMPI built from source + export OMPI_PREFIX=/path/to/ompi/install + export PATH=$OMPI_PREFIX/bin:$PATH + export LD_LIBRARY_PATH=$OMPI_PREFIX/lib:$LD_LIBRARY_PATH + +Step 2 — Ascent and Conduit +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Use the ``build_ascent.sh`` script provided in the Ascent repository +(``scripts/build_ascent/build_ascent.sh``). This script builds Conduit as an +internal dependency, guaranteeing version compatibility. The flags below cover +a full GPU+MPI+Python build suitable for use with PeleLMeX, PeleC, PyFR, and +nekRS. Adjust ``CUDA_ARCH`` and ``CUDA_ARCH_VTKM`` to match your GPU. :: + + env \ + enable_cuda=ON \ + CUDA_ARCH=89 \ + CUDA_ARCH_VTKM=ada \ + enable_mpi=ON \ + enable_mpicc=ON \ + enable_python=ON \ + enable_openmp=ON \ + enable_fortran=OFF \ + enable_tests=OFF \ + build_shared_libs=ON \ + prefix=/path/to/ascent/tpls \ + CC=gcc \ + CXX=g++ \ + MPICC=$OMPI_PREFIX/bin/mpicc \ + MPICXX=$OMPI_PREFIX/bin/mpicxx \ + MPIFC=$OMPI_PREFIX/bin/mpifort \ + ./scripts/build_ascent/build_ascent.sh + +This produces symlinks at ``/path/to/ascent/tpls/install/ascent-checkout`` and +``/path/to/ascent/tpls/install/conduit-v*``. For CPU-only builds, set +``enable_cuda=OFF`` and remove the ``CUDA_ARCH*`` variables. + +.. note:: + The following APT packages are required before running ``build_ascent.sh`` + on Ubuntu, or the build will fail at the Conduit or Viskores stage: :: + + sudo apt install -y \ + libglew-dev libegl1-mesa-dev libgl1-mesa-dev \ + python3-dev python3-numpy cython3 + +Step 3 — PeleLMeX +^^^^^^^^^^^^^^^^^^ + +Pass the Ascent and Conduit install paths to the GNUmake build. AMReX's GNUmake +system locates MPI via the compiler wrappers in ``PATH`` — ensure +``$OMPI_PREFIX/bin`` (or equivalent) is in your ``PATH`` before running +``make``, as set in Step 1. Combine with any physics flags appropriate for your +simulation: :: + + make -j8 \ + USE_MPI=TRUE \ + USE_CUDA=TRUE CUDA_ARCH=89 \ + USE_ASCENT=TRUE \ + ASCENT_DIR=/path/to/ascent/tpls/install/ascent-checkout \ + USE_CONDUIT=TRUE \ + CONDUIT_DIR=/path/to/ascent/tpls/install/conduit-v0.9.5 + +These flags can be combined with any physics flags (``USE_SOOT``, +``USE_RADIATION``, ``USE_PARTICLES``, ``USE_PLASMA``, ``USE_EB``). The Ascent +integration automatically publishes the additional fields for each compiled +physics module when the corresponding runtime flags are active. + +For CPU-only builds, omit ``USE_CUDA=TRUE`` and ``CUDA_ARCH``. + +.. note:: + ``LD_LIBRARY_PATH`` must include the Ascent, Conduit, and MPI library + directories at runtime, or the executable will fail to load shared + libraries. It is strongly recommended to set these in a persistent + environment script: :: + + export ASCENT_DIR=/path/to/ascent/tpls/install/ascent-checkout + export CONDUIT_DIR=/path/to/ascent/tpls/install/conduit-v0.9.5 + export LD_LIBRARY_PATH=$ASCENT_DIR/lib:$CONDUIT_DIR/lib:$LD_LIBRARY_PATH + +.. _sec:insitu::runtime: + +Runtime activation +------------------ + +Ascent is activated at runtime by adding the following to the input file: :: + + ascent.plot_int = 10 # call Ascent every 10 time steps + +When ``ascent.plot_int`` is not set or is negative, Ascent is fully disabled +with no runtime overhead. + +Ascent reads two YAML files from the run directory automatically: + +- ``ascent_actions.yaml`` — defines what to render (scenes, pipelines, filters). + This file is **required** for Ascent to produce any output. +- ``ascent_options.yaml`` — optional runtime configuration. The most common use + is to override the rendering backend: :: + + runtime: + viskores: + backend: openmp # valid values: cuda, openmp, serial, kokkos + + When ``ascent_options.yaml`` is absent or no backend is specified, Ascent + selects the highest-performance backend available in your build, using the + priority order CUDA → OpenMP → Kokkos → Serial. Which backends are available + depends on the flags passed to ``build_ascent.sh``: a build with + ``enable_cuda=ON`` will default to CUDA; a CPU-only build with + ``enable_openmp=ON`` will default to OpenMP. Override explicitly when you + want to free GPU memory for the solver (``backend: openmp``) or force a + specific device in a multi-GPU environment. + +For a full reference of available Ascent actions (contours, volume rendering, +Cinema databases, triggers, expressions, and more), see the +`Ascent actions documentation +`_. + +.. _sec:insitu::fields: + +Published fields +---------------- + +PeleLMeX passes the same field set to Ascent as it writes to plotfiles. +The tables below list every field available for rendering in +``ascent_actions.yaml``, grouped by the runtime flag or compile-time option +that controls their inclusion. Fields are always referred to by their exact +string name in the yaml ``field:`` key. + +Base state +^^^^^^^^^^ + +Always published. Species fields are controlled by ``amr.plot_speciesState``. + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name(s) + - Components + - Description + * - ``x_velocity``, ``y_velocity`` [, ``z_velocity``] + - SPACEDIM + - Velocity components + * - ``density`` + - 1 + - Mixture density :math:`\rho` + * - ``rho.Y()`` + - NUM_SPECIES + - Species partial densities :math:`\rho Y_k`. Published when + ``amr.plot_speciesState = 1`` (default). Set to ``0`` to suppress. + * - ``rhoh`` + - 1 + - Mixture enthalpy :math:`\rho h` + * - ``temp`` + - 1 + - Temperature + * - ``RhoRT`` + - 1 + - :math:`\rho R T` (thermodynamic pressure proxy) + * - ``divu`` + - 1 + - Velocity divergence constraint (when ``peleLM.has_divu = 1``) + * - ``gradpx`` [, ``gradpy``, ``gradpz``] + - SPACEDIM + - Pressure gradient components (when ``peleLM.plot_grad_p = 1``) + +Reaction rates +^^^^^^^^^^^^^^ + +Published when ``peleLM.do_react = 1`` (reacting flow) and +``peleLM.plot_react = 1`` (default when reacting). + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name(s) + - Components + - Description + * - ``I_R()`` + - NUM_SPECIES + - Species reaction rates :math:`\dot{\omega}_k` + * - ``FunctCall`` + - 1 + - CVODE integrator function call count per cell (chemistry stiffness diagnostic) + * - ``HeatRelease`` + - 1 + - Heat release rate. Published when additionally ``peleLM.plot_heatRelease = 1``. + +Derived variables +^^^^^^^^^^^^^^^^^ + +Published for each name listed in ``amr.derive_plot_vars``. The full list of +available derived variables and their descriptions is given in the +:doc:`LMeXControls` page under *PeleLMeX derived variables*. Commonly useful +for in-situ visualization: + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``avg_pressure`` + - 1 + - Cell-averaged pressure from nodal :math:`\pi` + * - ``mag_vort`` + - 1 + - Vorticity magnitude :math:`|\boldsymbol{\omega}|` + * - ``vorticity`` + - 1 (2D) / 3 (3D) + - Vorticity components + * - ``Qcrit`` + - 1 + - Q-criterion + * - ``kinetic_energy`` + - 1 + - :math:`\frac{1}{2} \rho |\mathbf{u}|^2` + * - ``enstrophy`` + - 1 + - :math:`\frac{1}{2} \rho |\boldsymbol{\omega}|^2` + * - ``viscosity`` + - 1 + - Mixture dynamic viscosity + * - ``mass_fractions`` + - NUM_SPECIES + - Species mass fractions ``Y()`` + * - ``mixture_fraction`` + - 1 + - Bilger mixture fraction (requires additional inputs, see :doc:`LMeXControls`) + * - ``progress_variable`` + - 1 + - Progress variable (requires additional inputs, see :doc:`LMeXControls`) + +LES turbulent viscosity +^^^^^^^^^^^^^^^^^^^^^^^ + +Published when ``peleLM.les_model`` is set to a non-``None`` model **and** +``peleLM.plot_les = 1``. Computing the turbulent viscosity at the plot time +requires a full velocity gradient tensor evaluation; set ``peleLM.plot_les = 0`` +to suppress this cost while retaining the resolved LES flow fields above. + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``viscturb`` + - 1 + - Turbulent (SGS) viscosity, face-to-cell interpolated + +Soot (``PELE_USE_SOOT``) +^^^^^^^^^^^^^^^^^^^^^^^^ + +Published when compiled with ``USE_SOOT=TRUE`` and ``peleLM.do_soot_solve = 1``. +Field names are assigned by the soot model from the moment indices; the exact +names are mechanism-dependent and can be discovered at runtime (see +:ref:`sec:insitu::discovery`). + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``soot_N``, ``soot_N0`` + - 1 each + - Total and nucleation soot number densities + * - ``soot_S`` + - 1 + - Soot surface area density + * - ``soot_fv`` + - 1 + - Soot volume fraction + +Radiation (``PELE_USE_RADIATION``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Published when compiled with ``USE_RADIATION=TRUE`` and +``peleLM.do_rad_solve = 1``. + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``rad.G`` + - 1 + - Mean radiative intensity :math:`G` + * - ``rad.kappa`` + - 1 + - Absorption coefficient :math:`\kappa` + * - ``rad.emis`` + - 1 + - Emission :math:`\kappa B` + +Spray (``PELE_USE_SPRAY``) +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Published when compiled with ``USE_PARTICLES=TRUE``. Spray derived quantities +are projected onto the AMR mesh grid via virtual particle interpolation. +Field names are assigned by the spray model and are mechanism-dependent; +the exact names can be discovered at runtime (see :ref:`sec:insitu::discovery`). +Typical fields include: + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``spray_num`` + - 1 + - Droplet number density (parcels per unit volume) + * - ``spray_mass`` + - 1 + - Spray mass density + * - ``spray_vol_frac`` + - 1 + - Liquid volume fraction + * - ``spray_temp`` + - 1 + - Droplet temperature + * - ``spray_density`` + - 1 + - Droplet material density + * - ``d10`` + - 1 + - Arithmetic mean diameter + * - ``d32`` + - 1 + - Sauter mean diameter + * - ``spray_x_vel``, ``spray_y_vel`` + - 1 each + - Droplet velocity components + * - ``wall_film_hght``, ``wall_film_mass`` + - 1 each + - Wall film height and mass (when wall film model is active) + +Plasma (``PELE_USE_PLASMA``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Published when compiled with ``USE_PLASMA=TRUE``. + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``nE`` + - 1 + - Electron number density + * - ``phiV`` + - 1 + - Electric potential + * - ``DriftFlux__X`` [``_Y``, ``_Z``] + - NUM_IONS × SPACEDIM + - Ion drift fluxes (when ``peleLM.do_extraEFdiags = 1``) + +Embedded boundary (``AMREX_USE_EB``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Published when compiled with ``USE_EB=TRUE``. + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``volFrac`` + - 1 + - EB volume fraction + +.. _sec:insitu::discovery: + +Discovering available field names at runtime +-------------------------------------------- + +Because some field names depend on the chemistry mechanism or physics model +(e.g. species names in ``rho.Y()`` and ``I_R()``, or soot +and spray model variable names), it is useful to have Ascent report exactly +which fields are present in the published mesh. This can be done by requesting +a nonexistent field in ``ascent_actions.yaml``: :: + + - + action: "add_scenes" + scenes: + s1: + plots: + p1: + type: "pseudocolor" + field: "DISCOVER_FIELDS" + renders: + r1: + image_prefix: "discover_%05d" + image_width: 512 + image_height: 512 + +Ascent will print a line of the form: :: + + (s1/p1) unknown field 'DISCOVER_FIELDS' field names: 'RhoRT', 'Y(N2)', + 'avg_pressure', 'density', 'soot_fv', 'soot_N', 'rad.G', ... + +listing every field available for that run. This is the canonical way to +obtain the exact field name strings for a given mechanism and physics +configuration before writing a production actions file. + +.. _sec:insitu::examples: + +Example actions files +--------------------- + +Pseudocolor temperature with AMR mesh overlay +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:: + + - + action: "add_scenes" + scenes: + scene1: + image_prefix: "temp_%05d" + plots: + plt1: + type: "pseudocolor" + field: "temp" + scene2: + image_prefix: "temp_mesh_%05d" + plots: + plt1: + type: "pseudocolor" + field: "temp" + plt2: + type: "mesh" + +Multiple fields in a single run +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Multiple scenes are rendered simultaneously at each in-situ call with no +additional solver cost — the mesh is published once and all scenes consume +it. :: + + - + action: "add_scenes" + scenes: + s1: + image_prefix: "temp_%05d" + plots: + p1: + type: "pseudocolor" + field: "temp" + s2: + image_prefix: "pressure_%05d" + plots: + p1: + type: "pseudocolor" + field: "avg_pressure" + s3: + image_prefix: "vorticity_%05d" + plots: + p1: + type: "pseudocolor" + field: "mag_vort" + +Reaction rate visualization (reacting cases) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Requires ``peleLM.plot_react = 1`` in the input file. :: + + - + action: "add_scenes" + scenes: + s1: + image_prefix: "heatrelease_%05d" + plots: + p1: + type: "pseudocolor" + field: "HeatRelease" + s2: + image_prefix: "IR_CH4_%05d" + plots: + p1: + type: "pseudocolor" + field: "I_R(CH4)" + +.. _sec:insitu::example_run: + +Example run +----------- + +To run the ``HotBubble`` case with in-situ rendering every 50 steps, +suppressing plotfile and checkpoint output: :: + + mpirun -n 1 ./PeleLMeX2d.gnu.MPI.CUDA.ex input.2d-regt \ + amr.max_step=400 amr.plot_int=-1 amr.check_int=-1 \ + ascent.plot_int=50 + +A reference ``ascent_actions.yaml`` is provided in the ``HotBubble`` case +directory at ``Exec/RegTests/HotBubble/ascent_actions.yaml``. diff --git a/Docs/sphinx/manual/Tutorials_HotBubble.rst b/Docs/sphinx/manual/Tutorials_HotBubble.rst index b85ff30d0..156432070 100644 --- a/Docs/sphinx/manual/Tutorials_HotBubble.rst +++ b/Docs/sphinx/manual/Tutorials_HotBubble.rst @@ -327,86 +327,19 @@ In-situ visualization with Ascent ---------------------------------- `PeleLMeX` supports in-situ visualization via `Ascent -`_, an open-source many-core capable -lightweight in-situ visualization and analysis library developed as part of the -`Alpine `_ project. Ascent uses `Conduit -`_ to describe and pass simulation data, -and the `Viskores `_ library for -rendering on both CPU and GPU. Since the solver state is passed directly to -Ascent without writing to disk, in-situ rendering eliminates the I/O bottleneck -of traditional post-hoc workflows and is well-suited to large-scale GPU runs. +`_, allowing fields to be rendered at runtime +without writing to disk. For full build instructions, runtime configuration, +available fields by physics module, and example actions files, see the +:doc:`InSituViz` page. -.. note:: - Ascent and Conduit must be built and installed before enabling in-situ - visualization. Refer to the `Ascent build documentation - `_ and the - `Conduit build documentation - `_ for - instructions. The ``build_ascent.sh`` script provided in the Ascent - repository (``scripts/build_ascent/build_ascent.sh``) builds both Ascent - and Conduit together and is the recommended approach. - -Building with Ascent -^^^^^^^^^^^^^^^^^^^^^ - -Ascent support is enabled at compile time by passing the following flags to -the GNUmake build system (the ``HotBubble`` GNUmakefile is unchanged): :: - - make -j8 USE_CUDA=TRUE CUDA_ARCH=89 \ - USE_ASCENT=TRUE \ - ASCENT_DIR=/path/to/ascent/install \ - USE_CONDUIT=TRUE \ - CONDUIT_DIR=/path/to/conduit/install - -Replace ``/path/to/ascent/install`` and ``/path/to/conduit/install`` with the -paths to your Ascent and Conduit installations. ``CUDA_ARCH`` should match your -GPU's compute capability (e.g., ``89`` for NVIDIA RTX 40-series, ``80`` for -A100). Omit the ``USE_CUDA`` flags to build a CPU-only Ascent-enabled -executable. - -Runtime activation -^^^^^^^^^^^^^^^^^^^ - -Ascent is activated at runtime by adding the following to the input file (or -passing on the command line): :: - - ascent.plot_int = 10 # call Ascent every 10 time steps - -When ``ascent.plot_int`` is not set or is negative, Ascent is disabled and -there is no runtime overhead. - -Ascent reads a YAML actions file named ``ascent_actions.yaml`` from the run -directory automatically. This file describes what to render. A reference file -is provided in the ``HotBubble`` case directory: :: - - Exec/RegTests/HotBubble/ascent_actions.yaml - -The reference file defines two scenes rendered simultaneously at each in-situ -call with no additional solver cost: :: - - - - action: "add_scenes" - scenes: - scene1: - image_prefix: "hotbubble_temp_%05d" - plots: - plt1: - type: "pseudocolor" - field: "temp" - scene2: - image_prefix: "hotbubble_mesh_%05d" - plots: - plt1: - type: "pseudocolor" - field: "temp" - plt2: - type: "mesh" - -``scene1`` renders the temperature field as a pseudocolor image. -``scene2`` renders the same temperature field with the AMR mesh overlaid, -clearly showing the block-structured refinement levels that `PeleLMeX` -automatically generates around the bubble interface. +To run the ``HotBubble`` case with in-situ rendering every 50 steps: :: + + mpirun -n 1 ./PeleLMeX2d.gnu.MPI.CUDA.ex input.2d-regt \ + amr.max_step=400 amr.plot_int=-1 amr.check_int=-1 \ + ascent.plot_int=50 +A reference ``ascent_actions.yaml`` rendering temperature and the AMR mesh +overlay is provided in ``Exec/RegTests/HotBubble/ascent_actions.yaml``. .. figure:: images/tutorials/HB_Ascent_combined.png :name: HB_Ascent_combined @@ -414,35 +347,3 @@ automatically generates around the bubble interface. :figwidth: 95% : Temperature field (left) and temperature with AMR mesh overlay (right) at step 200. - - -Ascent automatically selects the best available rendering backend — CUDA -when available, otherwise OpenMP — based on how it was compiled. No -additional configuration is required. - -Published fields -^^^^^^^^^^^^^^^^^ - -The following fields from the PeleLMeX state vector are published to Ascent -at each in-situ call and are available for rendering in ``ascent_actions.yaml``: - -* ``x_velocity``, ``y_velocity`` (``z_velocity`` in 3D) -* ``density`` -* ``rho.Y()`` for each species in the mechanism (e.g. ``rho.Y(N2)``) -* ``rhoh`` -* ``temp`` -* ``RhoRT`` - -Example run -^^^^^^^^^^^^ - -To run the ``HotBubble`` case with in-situ rendering every 50 steps: :: - - mpirun -n 1 ./PeleLMeX2d.gnu.MPI.CUDA.ex input.2d-regt \ - amr.max_step=400 amr.plot_int=-1 amr.check_int=-1 \ - ascent.plot_int=50 - - -For more information on available Ascent actions (contours, volume rendering, -Cinema databases, triggers, etc.), see the `Ascent actions documentation -`_. diff --git a/Docs/sphinx/manual/index.rst b/Docs/sphinx/manual/index.rst index cecaa0976..fb5e8dbec 100644 --- a/Docs/sphinx/manual/index.rst +++ b/Docs/sphinx/manual/index.rst @@ -31,6 +31,7 @@ point your web browser at the file ``${PELE_HOME}/Docs/build/html/index.html``. :caption: Usage: LMeXControls.rst + InSituViz.rst Troubleshooting.rst .. toctree:: From 9c240ba77f289922c416d9d4b1a2a1a91464baf5 Mon Sep 17 00:00:00 2001 From: AlexBogaev Date: Wed, 10 Jun 2026 19:51:29 -0400 Subject: [PATCH 4/8] Revise InSituViz.rst: restore original build depth, add MPI ABI warning, add HotBubble actions yaml --- Docs/sphinx/manual/InSituViz.rst | 149 +++++++-------------- Docs/sphinx/manual/Tutorials_HotBubble.rst | 25 +++- 2 files changed, 72 insertions(+), 102 deletions(-) diff --git a/Docs/sphinx/manual/InSituViz.rst b/Docs/sphinx/manual/InSituViz.rst index 0fac3f761..c3912df04 100644 --- a/Docs/sphinx/manual/InSituViz.rst +++ b/Docs/sphinx/manual/InSituViz.rst @@ -20,103 +20,56 @@ The PeleLMeX Ascent integration publishes exactly the same fields as ``WritePlotFile()``, controlled at runtime by the same input file flags. Any field visible in a plotfile is also available for in-situ rendering. -.. _sec:insitu::build: - -Building the full stack ------------------------ - -Ascent in-situ visualization requires that Ascent, Conduit, and PeleLMeX are -all built against the same MPI installation and, for GPU rendering, the same -CUDA toolkit. Building any component against a different MPI will cause ABI -mismatches at runtime. The recommended approach is to build the entire stack in -order: MPI first, then Ascent+Conduit via ``build_ascent.sh``, then PeleLMeX. - -Step 1 — MPI -^^^^^^^^^^^^ - -Build or install an MPI implementation. OpenMPI, MPICH, MVAPICH, Intel MPI, and -Cray MPI are all supported; Ascent uses the standard MPI-2 API and is not tied -to any specific implementation. Record the install prefix — it is needed for -every subsequent step. :: - - # Example: OpenMPI built from source - export OMPI_PREFIX=/path/to/ompi/install - export PATH=$OMPI_PREFIX/bin:$PATH - export LD_LIBRARY_PATH=$OMPI_PREFIX/lib:$LD_LIBRARY_PATH - -Step 2 — Ascent and Conduit -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Use the ``build_ascent.sh`` script provided in the Ascent repository -(``scripts/build_ascent/build_ascent.sh``). This script builds Conduit as an -internal dependency, guaranteeing version compatibility. The flags below cover -a full GPU+MPI+Python build suitable for use with PeleLMeX, PeleC, PyFR, and -nekRS. Adjust ``CUDA_ARCH`` and ``CUDA_ARCH_VTKM`` to match your GPU. :: - - env \ - enable_cuda=ON \ - CUDA_ARCH=89 \ - CUDA_ARCH_VTKM=ada \ - enable_mpi=ON \ - enable_mpicc=ON \ - enable_python=ON \ - enable_openmp=ON \ - enable_fortran=OFF \ - enable_tests=OFF \ - build_shared_libs=ON \ - prefix=/path/to/ascent/tpls \ - CC=gcc \ - CXX=g++ \ - MPICC=$OMPI_PREFIX/bin/mpicc \ - MPICXX=$OMPI_PREFIX/bin/mpicxx \ - MPIFC=$OMPI_PREFIX/bin/mpifort \ - ./scripts/build_ascent/build_ascent.sh - -This produces symlinks at ``/path/to/ascent/tpls/install/ascent-checkout`` and -``/path/to/ascent/tpls/install/conduit-v*``. For CPU-only builds, set -``enable_cuda=OFF`` and remove the ``CUDA_ARCH*`` variables. - .. note:: - The following APT packages are required before running ``build_ascent.sh`` - on Ubuntu, or the build will fail at the Conduit or Viskores stage: :: - - sudo apt install -y \ - libglew-dev libegl1-mesa-dev libgl1-mesa-dev \ - python3-dev python3-numpy cython3 - -Step 3 — PeleLMeX -^^^^^^^^^^^^^^^^^^ + Ascent and Conduit must be built and installed before enabling in-situ + visualization. Refer to the `Ascent build documentation + `_ and the + `Conduit build documentation + `_ for + instructions. The ``build_ascent.sh`` script provided in the Ascent + repository (``scripts/build_ascent/build_ascent.sh``) builds both Ascent + and Conduit together and is the recommended approach. -Pass the Ascent and Conduit install paths to the GNUmake build. AMReX's GNUmake -system locates MPI via the compiler wrappers in ``PATH`` — ensure -``$OMPI_PREFIX/bin`` (or equivalent) is in your ``PATH`` before running -``make``, as set in Step 1. Combine with any physics flags appropriate for your -simulation: :: - - make -j8 \ - USE_MPI=TRUE \ - USE_CUDA=TRUE CUDA_ARCH=89 \ - USE_ASCENT=TRUE \ - ASCENT_DIR=/path/to/ascent/tpls/install/ascent-checkout \ - USE_CONDUIT=TRUE \ - CONDUIT_DIR=/path/to/ascent/tpls/install/conduit-v0.9.5 - -These flags can be combined with any physics flags (``USE_SOOT``, -``USE_RADIATION``, ``USE_PARTICLES``, ``USE_PLASMA``, ``USE_EB``). The Ascent -integration automatically publishes the additional fields for each compiled -physics module when the corresponding runtime flags are active. - -For CPU-only builds, omit ``USE_CUDA=TRUE`` and ``CUDA_ARCH``. - -.. note:: - ``LD_LIBRARY_PATH`` must include the Ascent, Conduit, and MPI library - directories at runtime, or the executable will fail to load shared - libraries. It is strongly recommended to set these in a persistent - environment script: :: +.. _sec:insitu::build: - export ASCENT_DIR=/path/to/ascent/tpls/install/ascent-checkout - export CONDUIT_DIR=/path/to/ascent/tpls/install/conduit-v0.9.5 - export LD_LIBRARY_PATH=$ASCENT_DIR/lib:$CONDUIT_DIR/lib:$LD_LIBRARY_PATH +Building with Ascent +-------------------- + +Ascent support is enabled at compile time by passing the following flags to +the GNUmake build system. AMReX's GNUmake locates MPI via the compiler +wrappers in ``PATH``, so ensure your MPI installation's ``bin/`` directory +is in ``PATH`` before running ``make``: :: + + make -j8 USE_CUDA=TRUE CUDA_ARCH=89 \ + USE_ASCENT=TRUE \ + ASCENT_DIR=/path/to/ascent/install \ + USE_CONDUIT=TRUE \ + CONDUIT_DIR=/path/to/conduit/install + +Replace ``/path/to/ascent/install`` and ``/path/to/conduit/install`` with the +paths to your Ascent and Conduit installations. ``CUDA_ARCH`` should match your +GPU's compute capability (e.g., ``89`` for NVIDIA RTX 40-series, ``80`` for +A100). Omit the ``USE_CUDA`` flags to build a CPU-only Ascent-enabled +executable. + +These flags can be combined with any other physics flags (``USE_SOOT``, +``USE_RADIATION``, ``USE_PARTICLES``, ``USE_PLASMA``). The Ascent integration +automatically publishes the additional fields for each compiled physics module +when the corresponding runtime flags are active. + +.. warning:: + Ascent, Conduit, and PeleLMeX must all be built against the **same MPI + installation**. Building any component against a different MPI will cause + ABI mismatches and runtime failures when ``libascent_mpi.so`` or + ``libconduit_mpi.so`` are loaded. Verify before building: :: + + which mpicc # must point to your chosen MPI installation + mpicc --version # confirm the version matches across all components + + The MPI implementation itself does not matter — OpenMPI, MPICH, MVAPICH, + Intel MPI, and Cray MPI are all supported — but the same installation must + be used throughout. The ``LD_LIBRARY_PATH`` must also include the MPI, Ascent, + and Conduit library directories at runtime. .. _sec:insitu::runtime: @@ -469,13 +422,7 @@ Pseudocolor temperature with AMR mesh overlay - action: "add_scenes" scenes: - scene1: - image_prefix: "temp_%05d" - plots: - plt1: - type: "pseudocolor" - field: "temp" - scene2: + s1: image_prefix: "temp_mesh_%05d" plots: plt1: diff --git a/Docs/sphinx/manual/Tutorials_HotBubble.rst b/Docs/sphinx/manual/Tutorials_HotBubble.rst index 156432070..5b530b840 100644 --- a/Docs/sphinx/manual/Tutorials_HotBubble.rst +++ b/Docs/sphinx/manual/Tutorials_HotBubble.rst @@ -339,7 +339,30 @@ To run the ``HotBubble`` case with in-situ rendering every 50 steps: :: ascent.plot_int=50 A reference ``ascent_actions.yaml`` rendering temperature and the AMR mesh -overlay is provided in ``Exec/RegTests/HotBubble/ascent_actions.yaml``. +overlay is provided in ``Exec/RegTests/HotBubble/ascent_actions.yaml``: :: + + - + action: "add_scenes" + scenes: + scene1: + image_prefix: "hotbubble_temp_%05d" + plots: + plt1: + type: "pseudocolor" + field: "temp" + scene2: + image_prefix: "hotbubble_mesh_%05d" + plots: + plt1: + type: "pseudocolor" + field: "temp" + plt2: + type: "mesh" + +``scene1`` renders the temperature field as a pseudocolor image. +``scene2`` renders the same temperature field with the AMR mesh overlaid. +Both scenes are rendered simultaneously at each in-situ call with no +additional solver cost — the mesh is published once and consumed by all scenes. .. figure:: images/tutorials/HB_Ascent_combined.png :name: HB_Ascent_combined From a902b6951d0d6ca5ecb5a4b8731260f41d98fbfb Mon Sep 17 00:00:00 2001 From: AlexBogaev Date: Wed, 10 Jun 2026 20:25:39 -0400 Subject: [PATCH 5/8] Fix MPI ABI and CUDA_ARCH warnings in InSituViz.rst --- Docs/sphinx/manual/InSituViz.rst | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/Docs/sphinx/manual/InSituViz.rst b/Docs/sphinx/manual/InSituViz.rst index c3912df04..a2c86905a 100644 --- a/Docs/sphinx/manual/InSituViz.rst +++ b/Docs/sphinx/manual/InSituViz.rst @@ -61,15 +61,23 @@ when the corresponding runtime flags are active. Ascent, Conduit, and PeleLMeX must all be built against the **same MPI installation**. Building any component against a different MPI will cause ABI mismatches and runtime failures when ``libascent_mpi.so`` or - ``libconduit_mpi.so`` are loaded. Verify before building: :: + ``libconduit_mpi.so`` are loaded. Note that OpenMPI and MPICH-family + implementations (MPICH, MVAPICH, Intel MPI, Cray MPI) have incompatible + ABIs and cannot be mixed. Verify before building: :: which mpicc # must point to your chosen MPI installation mpicc --version # confirm the version matches across all components - The MPI implementation itself does not matter — OpenMPI, MPICH, MVAPICH, - Intel MPI, and Cray MPI are all supported — but the same installation must - be used throughout. The ``LD_LIBRARY_PATH`` must also include the MPI, Ascent, - and Conduit library directories at runtime. + For GPU builds, Ascent, Conduit, and PeleLMeX must also be compiled with + the **same ``CUDA_ARCH``**. Viskores device kernels are compiled for a + specific ``sm_XX`` target and will fail to load on a GPU that does not + support that architecture. Verify: :: + + nvcc --version # confirm toolkit version + nvidia-smi # confirm driver version and GPU compute capability + + The ``LD_LIBRARY_PATH`` must also include the MPI, Ascent, and Conduit + library directories at runtime. .. _sec:insitu::runtime: From 90e14b472451a8b2b9ab3bd772765aa46bf6154c Mon Sep 17 00:00:00 2001 From: AlexBogaev Date: Wed, 10 Jun 2026 20:37:24 -0400 Subject: [PATCH 6/8] Fix RST bold/code conflict in InSituViz.rst warning block --- Docs/sphinx/manual/InSituViz.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Docs/sphinx/manual/InSituViz.rst b/Docs/sphinx/manual/InSituViz.rst index a2c86905a..a99e65e10 100644 --- a/Docs/sphinx/manual/InSituViz.rst +++ b/Docs/sphinx/manual/InSituViz.rst @@ -58,8 +58,8 @@ automatically publishes the additional fields for each compiled physics module when the corresponding runtime flags are active. .. warning:: - Ascent, Conduit, and PeleLMeX must all be built against the **same MPI - installation**. Building any component against a different MPI will cause + Ascent, Conduit, and PeleLMeX must all be built against the same MPI + installation. Building any component against a different MPI will cause ABI mismatches and runtime failures when ``libascent_mpi.so`` or ``libconduit_mpi.so`` are loaded. Note that OpenMPI and MPICH-family implementations (MPICH, MVAPICH, Intel MPI, Cray MPI) have incompatible @@ -69,7 +69,7 @@ when the corresponding runtime flags are active. mpicc --version # confirm the version matches across all components For GPU builds, Ascent, Conduit, and PeleLMeX must also be compiled with - the **same ``CUDA_ARCH``**. Viskores device kernels are compiled for a + the same ``CUDA_ARCH``. Viskores device kernels are compiled for a specific ``sm_XX`` target and will fail to load on a GPU that does not support that architecture. Verify: :: @@ -433,10 +433,10 @@ Pseudocolor temperature with AMR mesh overlay s1: image_prefix: "temp_mesh_%05d" plots: - plt1: + p1: type: "pseudocolor" field: "temp" - plt2: + p2: type: "mesh" Multiple fields in a single run From 6ab2fe07f387def0fe90ee74db1bcdf11e26656c Mon Sep 17 00:00:00 2001 From: AlexBogaev Date: Thu, 11 Jun 2026 02:31:39 -0400 Subject: [PATCH 7/8] Publish initial condition to Ascent and fix ascent_options.yaml backend key PeleLMeX_Init.cpp: call doInSituViz() at m_nstep=0 alongside WritePlotFile() at both the fresh-start and restart init paths, mirroring PeleC main.cpp line 153. Previously Ascent only rendered at steps N, 2N, ... and the initial condition was never published. InSituViz.rst: correct ascent_options.yaml backend key from 'viskores' to 'vtkm'. Ascent 0.9.x checks runtime/vtkm/backend; the 'viskores' key was introduced in the develop branch following the VTKm->Viskores rebranding and is silently ignored on 0.9.x. Add a version note so users on newer Ascent builds know to switch keys. --- Docs/sphinx/manual/InSituViz.rst | 6 +++++- Source/PeleLMeX_Init.cpp | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Docs/sphinx/manual/InSituViz.rst b/Docs/sphinx/manual/InSituViz.rst index a99e65e10..d7a662fac 100644 --- a/Docs/sphinx/manual/InSituViz.rst +++ b/Docs/sphinx/manual/InSituViz.rst @@ -99,8 +99,12 @@ Ascent reads two YAML files from the run directory automatically: is to override the rendering backend: :: runtime: - viskores: + vtkm: backend: openmp # valid values: cuda, openmp, serial, kokkos + + .. note:: + The key is ``vtkm`` in Ascent 0.9.x. The develop branch uses ``viskores`` + following the library rebranding. When ``ascent_options.yaml`` is absent or no backend is specified, Ascent selects the highest-performance backend available in your build, using the diff --git a/Source/PeleLMeX_Init.cpp b/Source/PeleLMeX_Init.cpp index ea1288f39..dc8d8ac6c 100644 --- a/Source/PeleLMeX_Init.cpp +++ b/Source/PeleLMeX_Init.cpp @@ -281,6 +281,9 @@ PeleLM::initData() if (m_plot_int > 0 || m_plot_per_approx > 0. || m_plot_per_exact > 0.) { WritePlotFile(); } +#ifdef AMREX_USE_ASCENT + doInSituViz(); +#endif if (m_check_int > 0 || m_check_per > 0.) { WriteCheckPointFile(); } @@ -335,6 +338,9 @@ PeleLM::initData() if (m_plot_int > 0) { WritePlotFile(); } +#ifdef AMREX_USE_ASCENT + doInSituViz(); +#endif } #endif From 54eef0571ba91373df213b953063066af78bcd99 Mon Sep 17 00:00:00 2001 From: AlexBogaev Date: Sat, 13 Jun 2026 00:40:44 -0400 Subject: [PATCH 8/8] Address Copilot review comments on constructPlotMF() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PeleLMeX_Plot.cpp: - Fix incompressible ncomp: was unconditionally 2*SPACEDIM, now correctly SPACEDIM + SPACEDIM when m_plot_grad_p != 0. The old value was a latent bug in WritePlotFile() exposed by the new AMREX_ASSERT — gradp names are only pushed when m_plot_grad_p != 0 but ncomp always reserved space for them, causing the assert to trip for incompressible runs with default flags. - Fix spray name and fill blocks: add do_spray_particles guard alongside the existing NumDeriveVars() > 0 guard, matching the ncomp accounting block. Without this, name/fill could run when do_spray_particles=0 even if NumDeriveVars() is nonzero, desynchronizing ncomp vs a_plt_VarsName. InSituViz.rst: - Correct amr.plot_speciesState default: was documented as default=1 (always on), actual default is 0 (species suppressed unless explicitly enabled), consistent with LMeXControls.rst DEF=0. --- Docs/sphinx/manual/InSituViz.rst | 4 ++-- Source/PeleLMeX_Plot.cpp | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Docs/sphinx/manual/InSituViz.rst b/Docs/sphinx/manual/InSituViz.rst index d7a662fac..70c899395 100644 --- a/Docs/sphinx/manual/InSituViz.rst +++ b/Docs/sphinx/manual/InSituViz.rst @@ -134,7 +134,7 @@ string name in the yaml ``field:`` key. Base state ^^^^^^^^^^ -Always published. Species fields are controlled by ``amr.plot_speciesState``. +Always published. Species fields require ``amr.plot_speciesState = 1`` (default: ``0``). .. list-table:: :widths: 35 15 50 @@ -152,7 +152,7 @@ Always published. Species fields are controlled by ``amr.plot_speciesState``. * - ``rho.Y()`` - NUM_SPECIES - Species partial densities :math:`\rho Y_k`. Published when - ``amr.plot_speciesState = 1`` (default). Set to ``0`` to suppress. + ``amr.plot_speciesState = 1`` (default: ``0``). Set to ``1`` to include. * - ``rhoh`` - 1 - Mixture enthalpy :math:`\rho h` diff --git a/Source/PeleLMeX_Plot.cpp b/Source/PeleLMeX_Plot.cpp index 6273ce81c..52a300e27 100644 --- a/Source/PeleLMeX_Plot.cpp +++ b/Source/PeleLMeX_Plot.cpp @@ -142,8 +142,11 @@ PeleLM::constructPlotMF( // State if (m_incompressible != 0) { - // Velocity + pressure gradients - ncomp = 2 * AMREX_SPACEDIM; + // Velocity only, plus pressure gradients if requested + ncomp = AMREX_SPACEDIM; + if (m_plot_grad_p != 0) { + ncomp += AMREX_SPACEDIM; + } } else { // State + pressure gradients if (m_plot_grad_p != 0) { @@ -303,7 +306,7 @@ PeleLM::constructPlotMF( } } #ifdef PELE_USE_SPRAY - if (SprayParticleContainer::NumDeriveVars() > 0) { + if (do_spray_particles && SprayParticleContainer::NumDeriveVars() > 0) { // We need virtual particles for the lower levels setupVirtualParticles(0); for (const auto& spray_derive_name : @@ -449,7 +452,7 @@ PeleLM::constructPlotMF( cnt += mf->nComp(); } #ifdef PELE_USE_SPRAY - if (SprayParticleContainer::NumDeriveVars() > 0) { + if (do_spray_particles && SprayParticleContainer::NumDeriveVars() > 0) { const int num_spray_derive = SprayParticleContainer::NumDeriveVars(); a_mf_plt[lev].setVal(0., cnt, num_spray_derive); SprayPC->computeDerivedVars(a_mf_plt[lev], lev, cnt);