diff --git a/Docs/sphinx/manual/InSituViz.rst b/Docs/sphinx/manual/InSituViz.rst new file mode 100644 index 000000000..70c899395 --- /dev/null +++ b/Docs/sphinx/manual/InSituViz.rst @@ -0,0 +1,509 @@ +.. role:: cpp(code) + :language: c++ + +.. _sec:insitu: + +In-Situ Visualization with Ascent +================================== + +`PeleLMeX` supports in-situ visualization via `Ascent +`_, an open-source many-core capable +lightweight in-situ visualization and analysis library developed as part of the +`Alpine `_ project. Ascent uses `Conduit +`_ to describe and pass simulation data, +and the `Viskores `_ library for +rendering on both CPU and GPU. Since the solver state is passed directly to +Ascent without writing to disk, in-situ rendering eliminates the I/O bottleneck +of traditional post-hoc workflows and is well-suited to large-scale GPU runs. + +The PeleLMeX Ascent integration publishes exactly the same fields as +``WritePlotFile()``, controlled at runtime by the same input file flags. +Any field visible in a plotfile is also available for in-situ rendering. + +.. note:: + Ascent and Conduit must be built and installed before enabling in-situ + visualization. Refer to the `Ascent build documentation + `_ and the + `Conduit build documentation + `_ for + instructions. The ``build_ascent.sh`` script provided in the Ascent + repository (``scripts/build_ascent/build_ascent.sh``) builds both Ascent + and Conduit together and is the recommended approach. + +.. _sec:insitu::build: + +Building with Ascent +-------------------- + +Ascent support is enabled at compile time by passing the following flags to +the GNUmake build system. AMReX's GNUmake locates MPI via the compiler +wrappers in ``PATH``, so ensure your MPI installation's ``bin/`` directory +is in ``PATH`` before running ``make``: :: + + make -j8 USE_CUDA=TRUE CUDA_ARCH=89 \ + USE_ASCENT=TRUE \ + ASCENT_DIR=/path/to/ascent/install \ + USE_CONDUIT=TRUE \ + CONDUIT_DIR=/path/to/conduit/install + +Replace ``/path/to/ascent/install`` and ``/path/to/conduit/install`` with the +paths to your Ascent and Conduit installations. ``CUDA_ARCH`` should match your +GPU's compute capability (e.g., ``89`` for NVIDIA RTX 40-series, ``80`` for +A100). Omit the ``USE_CUDA`` flags to build a CPU-only Ascent-enabled +executable. + +These flags can be combined with any other physics flags (``USE_SOOT``, +``USE_RADIATION``, ``USE_PARTICLES``, ``USE_PLASMA``). The Ascent integration +automatically publishes the additional fields for each compiled physics module +when the corresponding runtime flags are active. + +.. warning:: + Ascent, Conduit, and PeleLMeX must all be built against the same MPI + installation. Building any component against a different MPI will cause + ABI mismatches and runtime failures when ``libascent_mpi.so`` or + ``libconduit_mpi.so`` are loaded. Note that OpenMPI and MPICH-family + implementations (MPICH, MVAPICH, Intel MPI, Cray MPI) have incompatible + ABIs and cannot be mixed. Verify before building: :: + + which mpicc # must point to your chosen MPI installation + mpicc --version # confirm the version matches across all components + + For GPU builds, Ascent, Conduit, and PeleLMeX must also be compiled with + the same ``CUDA_ARCH``. Viskores device kernels are compiled for a + specific ``sm_XX`` target and will fail to load on a GPU that does not + support that architecture. Verify: :: + + nvcc --version # confirm toolkit version + nvidia-smi # confirm driver version and GPU compute capability + + The ``LD_LIBRARY_PATH`` must also include the MPI, Ascent, and Conduit + library directories at runtime. + +.. _sec:insitu::runtime: + +Runtime activation +------------------ + +Ascent is activated at runtime by adding the following to the input file: :: + + ascent.plot_int = 10 # call Ascent every 10 time steps + +When ``ascent.plot_int`` is not set or is negative, Ascent is fully disabled +with no runtime overhead. + +Ascent reads two YAML files from the run directory automatically: + +- ``ascent_actions.yaml`` — defines what to render (scenes, pipelines, filters). + This file is **required** for Ascent to produce any output. +- ``ascent_options.yaml`` — optional runtime configuration. The most common use + is to override the rendering backend: :: + + runtime: + vtkm: + backend: openmp # valid values: cuda, openmp, serial, kokkos + + .. note:: + The key is ``vtkm`` in Ascent 0.9.x. The develop branch uses ``viskores`` + following the library rebranding. + + When ``ascent_options.yaml`` is absent or no backend is specified, Ascent + selects the highest-performance backend available in your build, using the + priority order CUDA → OpenMP → Kokkos → Serial. Which backends are available + depends on the flags passed to ``build_ascent.sh``: a build with + ``enable_cuda=ON`` will default to CUDA; a CPU-only build with + ``enable_openmp=ON`` will default to OpenMP. Override explicitly when you + want to free GPU memory for the solver (``backend: openmp``) or force a + specific device in a multi-GPU environment. + +For a full reference of available Ascent actions (contours, volume rendering, +Cinema databases, triggers, expressions, and more), see the +`Ascent actions documentation +`_. + +.. _sec:insitu::fields: + +Published fields +---------------- + +PeleLMeX passes the same field set to Ascent as it writes to plotfiles. +The tables below list every field available for rendering in +``ascent_actions.yaml``, grouped by the runtime flag or compile-time option +that controls their inclusion. Fields are always referred to by their exact +string name in the yaml ``field:`` key. + +Base state +^^^^^^^^^^ + +Always published. Species fields require ``amr.plot_speciesState = 1`` (default: ``0``). + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name(s) + - Components + - Description + * - ``x_velocity``, ``y_velocity`` [, ``z_velocity``] + - SPACEDIM + - Velocity components + * - ``density`` + - 1 + - Mixture density :math:`\rho` + * - ``rho.Y()`` + - NUM_SPECIES + - Species partial densities :math:`\rho Y_k`. Published when + ``amr.plot_speciesState = 1`` (default: ``0``). Set to ``1`` to include. + * - ``rhoh`` + - 1 + - Mixture enthalpy :math:`\rho h` + * - ``temp`` + - 1 + - Temperature + * - ``RhoRT`` + - 1 + - :math:`\rho R T` (thermodynamic pressure proxy) + * - ``divu`` + - 1 + - Velocity divergence constraint (when ``peleLM.has_divu = 1``) + * - ``gradpx`` [, ``gradpy``, ``gradpz``] + - SPACEDIM + - Pressure gradient components (when ``peleLM.plot_grad_p = 1``) + +Reaction rates +^^^^^^^^^^^^^^ + +Published when ``peleLM.do_react = 1`` (reacting flow) and +``peleLM.plot_react = 1`` (default when reacting). + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name(s) + - Components + - Description + * - ``I_R()`` + - NUM_SPECIES + - Species reaction rates :math:`\dot{\omega}_k` + * - ``FunctCall`` + - 1 + - CVODE integrator function call count per cell (chemistry stiffness diagnostic) + * - ``HeatRelease`` + - 1 + - Heat release rate. Published when additionally ``peleLM.plot_heatRelease = 1``. + +Derived variables +^^^^^^^^^^^^^^^^^ + +Published for each name listed in ``amr.derive_plot_vars``. The full list of +available derived variables and their descriptions is given in the +:doc:`LMeXControls` page under *PeleLMeX derived variables*. Commonly useful +for in-situ visualization: + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``avg_pressure`` + - 1 + - Cell-averaged pressure from nodal :math:`\pi` + * - ``mag_vort`` + - 1 + - Vorticity magnitude :math:`|\boldsymbol{\omega}|` + * - ``vorticity`` + - 1 (2D) / 3 (3D) + - Vorticity components + * - ``Qcrit`` + - 1 + - Q-criterion + * - ``kinetic_energy`` + - 1 + - :math:`\frac{1}{2} \rho |\mathbf{u}|^2` + * - ``enstrophy`` + - 1 + - :math:`\frac{1}{2} \rho |\boldsymbol{\omega}|^2` + * - ``viscosity`` + - 1 + - Mixture dynamic viscosity + * - ``mass_fractions`` + - NUM_SPECIES + - Species mass fractions ``Y()`` + * - ``mixture_fraction`` + - 1 + - Bilger mixture fraction (requires additional inputs, see :doc:`LMeXControls`) + * - ``progress_variable`` + - 1 + - Progress variable (requires additional inputs, see :doc:`LMeXControls`) + +LES turbulent viscosity +^^^^^^^^^^^^^^^^^^^^^^^ + +Published when ``peleLM.les_model`` is set to a non-``None`` model **and** +``peleLM.plot_les = 1``. Computing the turbulent viscosity at the plot time +requires a full velocity gradient tensor evaluation; set ``peleLM.plot_les = 0`` +to suppress this cost while retaining the resolved LES flow fields above. + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``viscturb`` + - 1 + - Turbulent (SGS) viscosity, face-to-cell interpolated + +Soot (``PELE_USE_SOOT``) +^^^^^^^^^^^^^^^^^^^^^^^^ + +Published when compiled with ``USE_SOOT=TRUE`` and ``peleLM.do_soot_solve = 1``. +Field names are assigned by the soot model from the moment indices; the exact +names are mechanism-dependent and can be discovered at runtime (see +:ref:`sec:insitu::discovery`). + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``soot_N``, ``soot_N0`` + - 1 each + - Total and nucleation soot number densities + * - ``soot_S`` + - 1 + - Soot surface area density + * - ``soot_fv`` + - 1 + - Soot volume fraction + +Radiation (``PELE_USE_RADIATION``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Published when compiled with ``USE_RADIATION=TRUE`` and +``peleLM.do_rad_solve = 1``. + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``rad.G`` + - 1 + - Mean radiative intensity :math:`G` + * - ``rad.kappa`` + - 1 + - Absorption coefficient :math:`\kappa` + * - ``rad.emis`` + - 1 + - Emission :math:`\kappa B` + +Spray (``PELE_USE_SPRAY``) +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Published when compiled with ``USE_PARTICLES=TRUE``. Spray derived quantities +are projected onto the AMR mesh grid via virtual particle interpolation. +Field names are assigned by the spray model and are mechanism-dependent; +the exact names can be discovered at runtime (see :ref:`sec:insitu::discovery`). +Typical fields include: + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``spray_num`` + - 1 + - Droplet number density (parcels per unit volume) + * - ``spray_mass`` + - 1 + - Spray mass density + * - ``spray_vol_frac`` + - 1 + - Liquid volume fraction + * - ``spray_temp`` + - 1 + - Droplet temperature + * - ``spray_density`` + - 1 + - Droplet material density + * - ``d10`` + - 1 + - Arithmetic mean diameter + * - ``d32`` + - 1 + - Sauter mean diameter + * - ``spray_x_vel``, ``spray_y_vel`` + - 1 each + - Droplet velocity components + * - ``wall_film_hght``, ``wall_film_mass`` + - 1 each + - Wall film height and mass (when wall film model is active) + +Plasma (``PELE_USE_PLASMA``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Published when compiled with ``USE_PLASMA=TRUE``. + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``nE`` + - 1 + - Electron number density + * - ``phiV`` + - 1 + - Electric potential + * - ``DriftFlux__X`` [``_Y``, ``_Z``] + - NUM_IONS × SPACEDIM + - Ion drift fluxes (when ``peleLM.do_extraEFdiags = 1``) + +Embedded boundary (``AMREX_USE_EB``) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Published when compiled with ``USE_EB=TRUE``. + +.. list-table:: + :widths: 35 15 50 + :header-rows: 1 + + * - Field name + - Components + - Description + * - ``volFrac`` + - 1 + - EB volume fraction + +.. _sec:insitu::discovery: + +Discovering available field names at runtime +-------------------------------------------- + +Because some field names depend on the chemistry mechanism or physics model +(e.g. species names in ``rho.Y()`` and ``I_R()``, or soot +and spray model variable names), it is useful to have Ascent report exactly +which fields are present in the published mesh. This can be done by requesting +a nonexistent field in ``ascent_actions.yaml``: :: + + - + action: "add_scenes" + scenes: + s1: + plots: + p1: + type: "pseudocolor" + field: "DISCOVER_FIELDS" + renders: + r1: + image_prefix: "discover_%05d" + image_width: 512 + image_height: 512 + +Ascent will print a line of the form: :: + + (s1/p1) unknown field 'DISCOVER_FIELDS' field names: 'RhoRT', 'Y(N2)', + 'avg_pressure', 'density', 'soot_fv', 'soot_N', 'rad.G', ... + +listing every field available for that run. This is the canonical way to +obtain the exact field name strings for a given mechanism and physics +configuration before writing a production actions file. + +.. _sec:insitu::examples: + +Example actions files +--------------------- + +Pseudocolor temperature with AMR mesh overlay +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:: + + - + action: "add_scenes" + scenes: + s1: + image_prefix: "temp_mesh_%05d" + plots: + p1: + type: "pseudocolor" + field: "temp" + p2: + type: "mesh" + +Multiple fields in a single run +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Multiple scenes are rendered simultaneously at each in-situ call with no +additional solver cost — the mesh is published once and all scenes consume +it. :: + + - + action: "add_scenes" + scenes: + s1: + image_prefix: "temp_%05d" + plots: + p1: + type: "pseudocolor" + field: "temp" + s2: + image_prefix: "pressure_%05d" + plots: + p1: + type: "pseudocolor" + field: "avg_pressure" + s3: + image_prefix: "vorticity_%05d" + plots: + p1: + type: "pseudocolor" + field: "mag_vort" + +Reaction rate visualization (reacting cases) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Requires ``peleLM.plot_react = 1`` in the input file. :: + + - + action: "add_scenes" + scenes: + s1: + image_prefix: "heatrelease_%05d" + plots: + p1: + type: "pseudocolor" + field: "HeatRelease" + s2: + image_prefix: "IR_CH4_%05d" + plots: + p1: + type: "pseudocolor" + field: "I_R(CH4)" + +.. _sec:insitu::example_run: + +Example run +----------- + +To run the ``HotBubble`` case with in-situ rendering every 50 steps, +suppressing plotfile and checkpoint output: :: + + mpirun -n 1 ./PeleLMeX2d.gnu.MPI.CUDA.ex input.2d-regt \ + amr.max_step=400 amr.plot_int=-1 amr.check_int=-1 \ + ascent.plot_int=50 + +A reference ``ascent_actions.yaml`` is provided in the ``HotBubble`` case +directory at ``Exec/RegTests/HotBubble/ascent_actions.yaml``. diff --git a/Docs/sphinx/manual/Tutorials_HotBubble.rst b/Docs/sphinx/manual/Tutorials_HotBubble.rst index b85ff30d0..5b530b840 100644 --- a/Docs/sphinx/manual/Tutorials_HotBubble.rst +++ b/Docs/sphinx/manual/Tutorials_HotBubble.rst @@ -327,62 +327,19 @@ In-situ visualization with Ascent ---------------------------------- `PeleLMeX` supports in-situ visualization via `Ascent -`_, an open-source many-core capable -lightweight in-situ visualization and analysis library developed as part of the -`Alpine `_ project. Ascent uses `Conduit -`_ to describe and pass simulation data, -and the `Viskores `_ library for -rendering on both CPU and GPU. Since the solver state is passed directly to -Ascent without writing to disk, in-situ rendering eliminates the I/O bottleneck -of traditional post-hoc workflows and is well-suited to large-scale GPU runs. +`_, allowing fields to be rendered at runtime +without writing to disk. For full build instructions, runtime configuration, +available fields by physics module, and example actions files, see the +:doc:`InSituViz` page. -.. note:: - Ascent and Conduit must be built and installed before enabling in-situ - visualization. Refer to the `Ascent build documentation - `_ and the - `Conduit build documentation - `_ for - instructions. The ``build_ascent.sh`` script provided in the Ascent - repository (``scripts/build_ascent/build_ascent.sh``) builds both Ascent - and Conduit together and is the recommended approach. - -Building with Ascent -^^^^^^^^^^^^^^^^^^^^^ - -Ascent support is enabled at compile time by passing the following flags to -the GNUmake build system (the ``HotBubble`` GNUmakefile is unchanged): :: - - make -j8 USE_CUDA=TRUE CUDA_ARCH=89 \ - USE_ASCENT=TRUE \ - ASCENT_DIR=/path/to/ascent/install \ - USE_CONDUIT=TRUE \ - CONDUIT_DIR=/path/to/conduit/install - -Replace ``/path/to/ascent/install`` and ``/path/to/conduit/install`` with the -paths to your Ascent and Conduit installations. ``CUDA_ARCH`` should match your -GPU's compute capability (e.g., ``89`` for NVIDIA RTX 40-series, ``80`` for -A100). Omit the ``USE_CUDA`` flags to build a CPU-only Ascent-enabled -executable. - -Runtime activation -^^^^^^^^^^^^^^^^^^^ - -Ascent is activated at runtime by adding the following to the input file (or -passing on the command line): :: - - ascent.plot_int = 10 # call Ascent every 10 time steps - -When ``ascent.plot_int`` is not set or is negative, Ascent is disabled and -there is no runtime overhead. - -Ascent reads a YAML actions file named ``ascent_actions.yaml`` from the run -directory automatically. This file describes what to render. A reference file -is provided in the ``HotBubble`` case directory: :: +To run the ``HotBubble`` case with in-situ rendering every 50 steps: :: - Exec/RegTests/HotBubble/ascent_actions.yaml + mpirun -n 1 ./PeleLMeX2d.gnu.MPI.CUDA.ex input.2d-regt \ + amr.max_step=400 amr.plot_int=-1 amr.check_int=-1 \ + ascent.plot_int=50 -The reference file defines two scenes rendered simultaneously at each in-situ -call with no additional solver cost: :: +A reference ``ascent_actions.yaml`` rendering temperature and the AMR mesh +overlay is provided in ``Exec/RegTests/HotBubble/ascent_actions.yaml``: :: - action: "add_scenes" @@ -403,10 +360,9 @@ call with no additional solver cost: :: type: "mesh" ``scene1`` renders the temperature field as a pseudocolor image. -``scene2`` renders the same temperature field with the AMR mesh overlaid, -clearly showing the block-structured refinement levels that `PeleLMeX` -automatically generates around the bubble interface. - +``scene2`` renders the same temperature field with the AMR mesh overlaid. +Both scenes are rendered simultaneously at each in-situ call with no +additional solver cost — the mesh is published once and consumed by all scenes. .. figure:: images/tutorials/HB_Ascent_combined.png :name: HB_Ascent_combined @@ -414,35 +370,3 @@ automatically generates around the bubble interface. :figwidth: 95% : Temperature field (left) and temperature with AMR mesh overlay (right) at step 200. - - -Ascent automatically selects the best available rendering backend — CUDA -when available, otherwise OpenMP — based on how it was compiled. No -additional configuration is required. - -Published fields -^^^^^^^^^^^^^^^^^ - -The following fields from the PeleLMeX state vector are published to Ascent -at each in-situ call and are available for rendering in ``ascent_actions.yaml``: - -* ``x_velocity``, ``y_velocity`` (``z_velocity`` in 3D) -* ``density`` -* ``rho.Y()`` for each species in the mechanism (e.g. ``rho.Y(N2)``) -* ``rhoh`` -* ``temp`` -* ``RhoRT`` - -Example run -^^^^^^^^^^^^ - -To run the ``HotBubble`` case with in-situ rendering every 50 steps: :: - - mpirun -n 1 ./PeleLMeX2d.gnu.MPI.CUDA.ex input.2d-regt \ - amr.max_step=400 amr.plot_int=-1 amr.check_int=-1 \ - ascent.plot_int=50 - - -For more information on available Ascent actions (contours, volume rendering, -Cinema databases, triggers, etc.), see the `Ascent actions documentation -`_. diff --git a/Docs/sphinx/manual/index.rst b/Docs/sphinx/manual/index.rst index cecaa0976..fb5e8dbec 100644 --- a/Docs/sphinx/manual/index.rst +++ b/Docs/sphinx/manual/index.rst @@ -31,6 +31,7 @@ point your web browser at the file ``${PELE_HOME}/Docs/build/html/index.html``. :caption: Usage: LMeXControls.rst + InSituViz.rst Troubleshooting.rst .. toctree:: diff --git a/Source/PeleLMeX.H b/Source/PeleLMeX.H index dad8bb0f5..56b21e4e2 100644 --- a/Source/PeleLMeX.H +++ b/Source/PeleLMeX.H @@ -1019,6 +1019,9 @@ public: //----------------------------------------------------------------------------- // I/O void WritePlotFile(); + void constructPlotMF( + amrex::Vector& a_mf_plt, + amrex::Vector& a_plt_VarsName); bool writePlotNow() const; bool checkMessage(const std::string& a_action) const; void WriteCheckPointFile(); diff --git a/Source/PeleLMeX_InSituViz.cpp b/Source/PeleLMeX_InSituViz.cpp index c4345110e..8c716b066 100644 --- a/Source/PeleLMeX_InSituViz.cpp +++ b/Source/PeleLMeX_InSituViz.cpp @@ -19,52 +19,33 @@ PeleLM::doInSituViz() auto dPlotFileTime0 = amrex::second(); - averageDownState(AmrNewTime); - - amrex::Vector speciesNames; - pele::physics::eos::speciesNames( - speciesNames, &(eos_parms.host_parm())); - - amrex::Vector plt_var_names; - AMREX_D_TERM(plt_var_names.push_back("x_velocity"); - , plt_var_names.push_back("y_velocity"); - , plt_var_names.push_back("z_velocity")); - plt_var_names.push_back("density"); - for (int n = 0; n < NUM_SPECIES; ++n) { - plt_var_names.push_back("rho.Y(" + speciesNames[n] + ")"); - } - plt_var_names.push_back("rhoh"); - plt_var_names.push_back("temp"); - plt_var_names.push_back("RhoRT"); - - // Base state: velocity + density + species + rhoh + temp + RhoRT - const int ncomp = RHORT + 1; - AMREX_ASSERT(ncomp == static_cast(plt_var_names.size())); - - const int nlevels = finest_level + 1; + // Assemble the same plot MultiFab and variable list used by WritePlotFile(). + // This ensures Ascent publishes all user-requested fields (species flag, + // plasma, soot, radiation, reactions, derived vars, spray, LES viscosity, + // ODE quantities, EB volume fraction, external sources) rather than only + // the hardcoded base state. averageDown is performed inside constructPlotMF. + // constructPlotMF also calls setupVirtualParticles(0) when PELE_USE_SPRAY is + // active; we are responsible for the matching removeVirtualParticles cleanup + // since there is no SprayParticleIO step here (unlike WritePlotFile). amrex::Vector plotMFs; - plotMFs.reserve(nlevels); - for (int lev = 0; lev <= finest_level; ++lev) { - plotMFs.emplace_back( - grids[lev], dmap[lev], ncomp, 0, amrex::MFInfo(), Factory(lev)); - amrex::MultiFab::Copy( - plotMFs[lev], m_leveldata_new[lev]->state, 0, 0, ncomp, 0); - } + amrex::Vector plt_var_names; + constructPlotMF(plotMFs, plt_var_names); - amrex::Vector plotMFs_constvec; - plotMFs_constvec.reserve(nlevels); - for (int lev = 0; lev < nlevels; ++lev) { - plotMFs_constvec.push_back( - static_cast(&plotMFs[lev])); +#ifdef PELE_USE_SPRAY + if (do_spray_particles) { + for (int lev = 0; lev <= finest_level; ++lev) { + removeVirtualParticles(lev); + } } +#endif // PeleLMeX is non-subcycling: all levels share m_nstep - amrex::Vector istep(nlevels, m_nstep); + amrex::Vector istep(finest_level + 1, m_nstep); conduit::Node bp_mesh; amrex::MultiLevelToBlueprint( - nlevels, plotMFs_constvec, plt_var_names, Geom(), m_cur_time, istep, - refRatio(), bp_mesh); + finest_level + 1, GetVecOfConstPtrs(plotMFs), plt_var_names, Geom(), + m_cur_time, istep, refRatio(), bp_mesh); ascent::Ascent ascent; conduit::Node open_opts; diff --git a/Source/PeleLMeX_Init.cpp b/Source/PeleLMeX_Init.cpp index ea1288f39..dc8d8ac6c 100644 --- a/Source/PeleLMeX_Init.cpp +++ b/Source/PeleLMeX_Init.cpp @@ -281,6 +281,9 @@ PeleLM::initData() if (m_plot_int > 0 || m_plot_per_approx > 0. || m_plot_per_exact > 0.) { WritePlotFile(); } +#ifdef AMREX_USE_ASCENT + doInSituViz(); +#endif if (m_check_int > 0 || m_check_per > 0.) { WriteCheckPointFile(); } @@ -335,6 +338,9 @@ PeleLM::initData() if (m_plot_int > 0) { WritePlotFile(); } +#ifdef AMREX_USE_ASCENT + doInSituViz(); +#endif } #endif diff --git a/Source/PeleLMeX_Plot.cpp b/Source/PeleLMeX_Plot.cpp index 7d4c72b58..52a300e27 100644 --- a/Source/PeleLMeX_Plot.cpp +++ b/Source/PeleLMeX_Plot.cpp @@ -83,13 +83,55 @@ PeleLM::WritePlotFile() amrex::VisMF::SetNOutFiles(m_nfiles); + //---------------------------------------------------------------- + // Average down the state + + amrex::Vector mf_plt; + amrex::Vector plt_VarsName; + constructPlotMF(mf_plt, plt_VarsName); + + // No SubCycling, all levels the same step. + amrex::Vector istep(finest_level + 1, m_nstep); + +#ifdef AMREX_USE_HDF5 + if (m_write_hdf5_pltfile) { + amrex::WriteMultiLevelPlotfileHDF5( + plotfilename, finest_level + 1, GetVecOfConstPtrs(mf_plt), plt_VarsName, + Geom(), m_cur_time, istep, refRatio()); + } else +#endif + { + amrex::WriteMultiLevelPlotfile( + plotfilename, finest_level + 1, GetVecOfConstPtrs(mf_plt), plt_VarsName, + Geom(), m_cur_time, istep, refRatio()); + } + +#ifdef PELE_USE_SPRAY + if (do_spray_particles) { + constexpr bool is_spraycheck = false; + for (int lev = 0; lev <= finest_level; ++lev) { + SprayPC->SprayParticleIO(lev, is_spraycheck, plotfilename); + // Remove virtual particles that were made for derived variables + removeVirtualParticles(lev); + } + } +#endif +} + +void +PeleLM::constructPlotMF( + amrex::Vector& a_mf_plt, + amrex::Vector& a_plt_VarsName) +{ + BL_PROFILE("PeleLMeX::constructPlotMF()"); + //---------------------------------------------------------------- // Average down the state averageDownState(AmrNewTime); if (m_nAux > 0) { averageDownAux(AmrNewTime); } - // Get consistent reaction data across level + // Get consistent reaction data across levels if ((m_do_react != 0) && (m_skipInstantRR == 0) && (m_plot_react != 0)) { averageDownReaction(); } @@ -100,8 +142,11 @@ PeleLM::WritePlotFile() // State if (m_incompressible != 0) { - // Velocity + pressure gradients - ncomp = 2 * AMREX_SPACEDIM; + // Velocity only, plus pressure gradients if requested + ncomp = AMREX_SPACEDIM; + if (m_plot_grad_p != 0) { + ncomp += AMREX_SPACEDIM; + } } else { // State + pressure gradients if (m_plot_grad_p != 0) { @@ -176,108 +221,108 @@ PeleLM::WritePlotFile() } //---------------------------------------------------------------- - // Plot MultiFabs - amrex::Vector mf_plt; - mf_plt.reserve(finest_level + 1); + // Allocate plot MultiFabs + a_mf_plt.clear(); + a_mf_plt.reserve(finest_level + 1); for (int lev = 0; lev <= finest_level; ++lev) { - mf_plt.emplace_back( + a_mf_plt.emplace_back( grids[lev], dmap[lev], ncomp, 0, amrex::MFInfo(), Factory(lev)); } //---------------------------------------------------------------- - // Components names + // Component names amrex::Vector names; pele::physics::eos::speciesNames( names, &(eos_parms.host_parm())); - amrex::Vector plt_VarsName; - AMREX_D_TERM(plt_VarsName.push_back("x_velocity"); - , plt_VarsName.push_back("y_velocity"); - , plt_VarsName.push_back("z_velocity")); + a_plt_VarsName.clear(); + AMREX_D_TERM(a_plt_VarsName.push_back("x_velocity"); + , a_plt_VarsName.push_back("y_velocity"); + , a_plt_VarsName.push_back("z_velocity")); if (m_incompressible == 0) { - plt_VarsName.push_back("density"); + a_plt_VarsName.push_back("density"); if (m_plotStateSpec != 0) { for (int n = 0; n < NUM_SPECIES; ++n) { - plt_VarsName.push_back("rho.Y(" + names[n] + ")"); + a_plt_VarsName.push_back("rho.Y(" + names[n] + ")"); } } - plt_VarsName.push_back("rhoh"); - plt_VarsName.push_back("temp"); - plt_VarsName.push_back("RhoRT"); + a_plt_VarsName.push_back("rhoh"); + a_plt_VarsName.push_back("temp"); + a_plt_VarsName.push_back("RhoRT"); #ifdef PELE_USE_PLASMA - plt_VarsName.push_back("nE"); - plt_VarsName.push_back("phiV"); + a_plt_VarsName.push_back("nE"); + a_plt_VarsName.push_back("phiV"); #endif #ifdef PELE_USE_SOOT for (int mom = 0; mom < NUMSOOTVAR; ++mom) { const std::string sootname = soot_model->sootVariableName(mom); - plt_VarsName.push_back(sootname); + a_plt_VarsName.push_back(sootname); } #endif #ifdef PELE_USE_RADIATION if (do_rad_solve) { - plt_VarsName.push_back("rad.G"); - plt_VarsName.push_back("rad.kappa"); - plt_VarsName.push_back("rad.emis"); + a_plt_VarsName.push_back("rad.G"); + a_plt_VarsName.push_back("rad.kappa"); + a_plt_VarsName.push_back("rad.emis"); } #endif if (m_has_divu != 0) { - plt_VarsName.push_back("divu"); + a_plt_VarsName.push_back("divu"); } } if (m_plot_grad_p != 0) { - AMREX_D_TERM(plt_VarsName.push_back("gradpx"); - , plt_VarsName.push_back("gradpy"); - , plt_VarsName.push_back("gradpz")); + AMREX_D_TERM(a_plt_VarsName.push_back("gradpx"); + , a_plt_VarsName.push_back("gradpy"); + , a_plt_VarsName.push_back("gradpz")); } for (int n = 0; n < m_nAux; ++n) { - plt_VarsName.push_back(m_aux_names[n]); + a_plt_VarsName.push_back(m_aux_names[n]); } if ((m_do_react != 0) && (m_skipInstantRR == 0) && (m_plot_react != 0)) { for (int n = 0; n < NUM_SPECIES; ++n) { - plt_VarsName.push_back("I_R(" + names[n] + ")"); + a_plt_VarsName.push_back("I_R(" + names[n] + ")"); } #ifdef PELE_USE_PLASMA - plt_VarsName.push_back("I_R(nE)"); + a_plt_VarsName.push_back("I_R(nE)"); #endif - plt_VarsName.push_back("FunctCall"); + a_plt_VarsName.push_back("FunctCall"); // Extras: if (m_plotHeatRelease != 0) { - plt_VarsName.push_back("HeatRelease"); + a_plt_VarsName.push_back("HeatRelease"); } } #ifdef AMREX_USE_EB - plt_VarsName.push_back("volFrac"); + a_plt_VarsName.push_back("volFrac"); #endif for (int ivar = 0; ivar < m_derivePlotVarCount; ++ivar) { const PeleLMDeriveRec* rec = derive_lst.get(m_derivePlotVars[ivar]); for (int dvar = 0; dvar < rec->numDerive(); ++dvar) { - plt_VarsName.push_back(rec->variableName(dvar)); + a_plt_VarsName.push_back(rec->variableName(dvar)); } } #ifdef PELE_USE_SPRAY - if (SprayParticleContainer::NumDeriveVars() > 0) { + if (do_spray_particles && SprayParticleContainer::NumDeriveVars() > 0) { // We need virtual particles for the lower levels setupVirtualParticles(0); for (const auto& spray_derive_name : SprayParticleContainer::DeriveVarNames()) { - plt_VarsName.push_back(spray_derive_name); + a_plt_VarsName.push_back(spray_derive_name); } } if (do_spray_particles && SprayParticleContainer::plot_spray_src) { - plt_VarsName.push_back("spray_mass_src"); - plt_VarsName.push_back("spray_energy_src"); - AMREX_D_TERM(plt_VarsName.push_back("spray_momentumX_src"); - , plt_VarsName.push_back("spray_momentumY_src"); - , plt_VarsName.push_back("spray_momentumZ_src")); + a_plt_VarsName.push_back("spray_mass_src"); + a_plt_VarsName.push_back("spray_energy_src"); + AMREX_D_TERM(a_plt_VarsName.push_back("spray_momentumX_src"); + , a_plt_VarsName.push_back("spray_momentumY_src"); + , a_plt_VarsName.push_back("spray_momentumZ_src")); for (const auto& spray_fuel_name : SprayParticleContainer::m_sprayDepNames) { - plt_VarsName.push_back("spray_" + spray_fuel_name + "_src"); + a_plt_VarsName.push_back("spray_" + spray_fuel_name + "_src"); } } #endif @@ -287,7 +332,7 @@ PeleLM::WritePlotFile() for (int ivar = 0; ivar < NUM_IONS; ++ivar) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const std::string dir = (idim == 0) ? "X" : ((idim == 1) ? "Y" : "Z"); - plt_VarsName.push_back( + a_plt_VarsName.push_back( "DriftFlux_" + names[NUM_SPECIES - NUM_IONS + ivar] + "_" + dir); } } @@ -295,19 +340,19 @@ PeleLM::WritePlotFile() #endif if (m_do_les && m_plot_les) { - plt_VarsName.push_back("viscturb"); + a_plt_VarsName.push_back("viscturb"); } #if NUM_ODE > 0 for (int n = 0; n < NUM_ODE; ++n) { - plt_VarsName.push_back(m_ode_names[n]); + a_plt_VarsName.push_back(m_ode_names[n]); } #endif // External source terms if (m_plot_extSource) { for (int ivar = 0; ivar < NVAR; ++ivar) { - plt_VarsName.push_back("extsource_" + stateVariableName(ivar)); + a_plt_VarsName.push_back("extsource_" + stateVariableName(ivar)); } } @@ -317,131 +362,133 @@ PeleLM::WritePlotFile() int cnt = 0; if (m_incompressible != 0) { amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, 0, cnt, AMREX_SPACEDIM, 0); + a_mf_plt[lev], m_leveldata_new[lev]->state, 0, cnt, AMREX_SPACEDIM, 0); cnt += AMREX_SPACEDIM; } else { // Velocity and density amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, 0, cnt, AMREX_SPACEDIM + 1, + a_mf_plt[lev], m_leveldata_new[lev]->state, 0, cnt, AMREX_SPACEDIM + 1, 0); cnt += AMREX_SPACEDIM + 1; // Species only if requested if (m_plotStateSpec != 0) { amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, FIRSTSPEC, cnt, NUM_SPECIES, - 0); + a_mf_plt[lev], m_leveldata_new[lev]->state, FIRSTSPEC, cnt, + NUM_SPECIES, 0); cnt += NUM_SPECIES; } amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, RHOH, cnt, 3, 0); + a_mf_plt[lev], m_leveldata_new[lev]->state, RHOH, cnt, 3, 0); cnt += 3; #ifdef PELE_USE_PLASMA amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, NE, cnt, 2, 0); + a_mf_plt[lev], m_leveldata_new[lev]->state, NE, cnt, 2, 0); cnt += 2; #endif #ifdef PELE_USE_SOOT amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, FIRSTSOOT, cnt, NUMSOOTVAR, + a_mf_plt[lev], m_leveldata_new[lev]->state, FIRSTSOOT, cnt, NUMSOOTVAR, 0); cnt += NUMSOOTVAR; #endif #ifdef PELE_USE_RADIATION if (do_rad_solve) { - amrex::MultiFab::Copy(mf_plt[lev], rad_model->G()[lev], 0, cnt, 1, 0); + amrex::MultiFab::Copy(a_mf_plt[lev], rad_model->G()[lev], 0, cnt, 1, 0); cnt += 1; amrex::MultiFab::Copy( - mf_plt[lev], rad_model->kappa()[lev], 0, cnt, 1, 0); + a_mf_plt[lev], rad_model->kappa()[lev], 0, cnt, 1, 0); cnt += 1; amrex::MultiFab::Copy( - mf_plt[lev], rad_model->emis()[lev], 0, cnt, 1, 0); + a_mf_plt[lev], rad_model->emis()[lev], 0, cnt, 1, 0); cnt += 1; } #endif if (m_has_divu != 0) { amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->divu, 0, cnt, 1, 0); + a_mf_plt[lev], m_leveldata_new[lev]->divu, 0, cnt, 1, 0); cnt += 1; } } if (m_plot_grad_p != 0) { amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->gp, 0, cnt, AMREX_SPACEDIM, 0); + a_mf_plt[lev], m_leveldata_new[lev]->gp, 0, cnt, AMREX_SPACEDIM, 0); cnt += AMREX_SPACEDIM; } if (m_nAux > 0) { amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->auxiliaries, 0, cnt, m_nAux, 0); + a_mf_plt[lev], m_leveldata_new[lev]->auxiliaries, 0, cnt, m_nAux, 0); cnt += m_nAux; } if ((m_do_react != 0) && (m_skipInstantRR == 0) && (m_plot_react != 0)) { amrex::MultiFab::Copy( - mf_plt[lev], m_leveldatareact[lev]->I_R, 0, cnt, nCompIR(), 0); + a_mf_plt[lev], m_leveldatareact[lev]->I_R, 0, cnt, nCompIR(), 0); cnt += nCompIR(); amrex::MultiFab::Copy( - mf_plt[lev], m_leveldatareact[lev]->functC, 0, cnt, 1, 0); + a_mf_plt[lev], m_leveldatareact[lev]->functC, 0, cnt, 1, 0); cnt += 1; if (m_plotHeatRelease != 0) { std::unique_ptr mf; mf = std::make_unique(grids[lev], dmap[lev], 1, 0); getHeatRelease(lev, mf.get()); - amrex::MultiFab::Copy(mf_plt[lev], *mf, 0, cnt, 1, 0); + amrex::MultiFab::Copy(a_mf_plt[lev], *mf, 0, cnt, 1, 0); cnt += 1; } } #ifdef AMREX_USE_EB amrex::MultiFab::Copy( - mf_plt[lev], EBFactory(lev).getVolFrac(), 0, cnt, 1, 0); + a_mf_plt[lev], EBFactory(lev).getVolFrac(), 0, cnt, 1, 0); cnt += 1; #endif for (int ivar = 0; ivar < m_derivePlotVarCount; ++ivar) { std::unique_ptr mf; mf = derive(m_derivePlotVars[ivar], m_cur_time, lev, 0); - amrex::MultiFab::Copy(mf_plt[lev], *mf, 0, cnt, mf->nComp(), 0); + amrex::MultiFab::Copy(a_mf_plt[lev], *mf, 0, cnt, mf->nComp(), 0); cnt += mf->nComp(); } #ifdef PELE_USE_SPRAY - if (SprayParticleContainer::NumDeriveVars() > 0) { + if (do_spray_particles && SprayParticleContainer::NumDeriveVars() > 0) { const int num_spray_derive = SprayParticleContainer::NumDeriveVars(); - mf_plt[lev].setVal(0., cnt, num_spray_derive); - SprayPC->computeDerivedVars(mf_plt[lev], lev, cnt); + a_mf_plt[lev].setVal(0., cnt, num_spray_derive); + SprayPC->computeDerivedVars(a_mf_plt[lev], lev, cnt); if (lev < finest_level) { amrex::MultiFab tmp_plt( grids[lev], dmap[lev], num_spray_derive, 0, amrex::MFInfo(), Factory(lev)); tmp_plt.setVal(0.); VirtPC->computeDerivedVars(tmp_plt, lev, 0); - amrex::MultiFab::Add(mf_plt[lev], tmp_plt, 0, cnt, num_spray_derive, 0); + amrex::MultiFab::Add( + a_mf_plt[lev], tmp_plt, 0, cnt, num_spray_derive, 0); } cnt += num_spray_derive; } if (do_spray_particles && SprayParticleContainer::plot_spray_src) { SprayComps scomps = SprayParticleContainer::getSprayComps(); amrex::MultiFab::Copy( - mf_plt[lev], *m_spraysource[lev], scomps.rhoSrcIndx, cnt++, 1, 0); + a_mf_plt[lev], *m_spraysource[lev], scomps.rhoSrcIndx, cnt++, 1, 0); amrex::MultiFab::Copy( - mf_plt[lev], *m_spraysource[lev], scomps.engSrcIndx, cnt++, 1, 0); + a_mf_plt[lev], *m_spraysource[lev], scomps.engSrcIndx, cnt++, 1, 0); amrex::MultiFab::Copy( - mf_plt[lev], *m_spraysource[lev], scomps.momSrcIndx, cnt, + a_mf_plt[lev], *m_spraysource[lev], scomps.momSrcIndx, cnt, AMREX_SPACEDIM, 0); cnt += AMREX_SPACEDIM; for (int spf = 0; spf < SPRAY_FUEL_NUM; ++spf) { amrex::MultiFab::Copy( - mf_plt[lev], *m_spraysource[lev], scomps.specSrcIndx + spf, cnt++, 1, - 0); + a_mf_plt[lev], *m_spraysource[lev], scomps.specSrcIndx + spf, cnt++, + 1, 0); } } #endif #ifdef PELE_USE_PLASMA if (m_do_extraEFdiags) { amrex::MultiFab::Copy( - mf_plt[lev], *m_ionsFluxes[lev], 0, cnt, m_ionsFluxes[lev]->nComp(), 0); + a_mf_plt[lev], *m_ionsFluxes[lev], 0, cnt, m_ionsFluxes[lev]->nComp(), + 0); cnt += m_ionsFluxes[lev]->nComp(); } #endif @@ -449,7 +496,7 @@ PeleLM::WritePlotFile() if (m_do_les && m_plot_les) { constexpr amrex::Real fact = 0.5 / static_cast(AMREX_SPACEDIM); - auto const& plot_arr = mf_plt[lev].arrays(); + auto const& plot_arr = a_mf_plt[lev].arrays(); AMREX_D_TERM( auto const& mut_arr_x = m_leveldata_new[lev]->visc_turb_fc[0].const_arrays(); @@ -459,10 +506,10 @@ PeleLM::WritePlotFile() m_leveldata_new[lev]->visc_turb_fc[2].const_arrays();) // interpolate turbulent viscosity from faces to centers amrex::ParallelFor( - mf_plt[lev], [plot_arr, cnt, mut_arr_x, mut_arr_y + a_mf_plt[lev], [plot_arr, cnt, mut_arr_x, mut_arr_y #if (AMREX_SPACEDIM == 3) - , - mut_arr_z + , + mut_arr_z #endif ] AMREX_GPU_DEVICE(int box_no, int i, int j, int k) noexcept { plot_arr[box_no](i, j, k, cnt) = @@ -478,49 +525,26 @@ PeleLM::WritePlotFile() #if NUM_ODE > 0 amrex::MultiFab::Copy( - mf_plt[lev], m_leveldata_new[lev]->state, FIRSTODE, cnt, NUM_ODE, 0); + a_mf_plt[lev], m_leveldata_new[lev]->state, FIRSTODE, cnt, NUM_ODE, 0); cnt += NUM_ODE; #endif if (m_plot_extSource) { - amrex::MultiFab::Copy(mf_plt[lev], *m_extSource[lev], 0, cnt, NVAR, 0); + amrex::MultiFab::Copy(a_mf_plt[lev], *m_extSource[lev], 0, cnt, NVAR, 0); } #ifdef AMREX_USE_EB if (m_plot_zeroEBcovered != 0) { - EB_set_covered(mf_plt[lev], 0.0); + EB_set_covered(a_mf_plt[lev], 0.0); } #endif } - // No SubCycling, all levels the same step. - amrex::Vector istep(finest_level + 1, m_nstep); - -#ifdef AMREX_USE_HDF5 - if (m_write_hdf5_pltfile) { - amrex::WriteMultiLevelPlotfileHDF5( - plotfilename, finest_level + 1, GetVecOfConstPtrs(mf_plt), plt_VarsName, - Geom(), m_cur_time, istep, refRatio()); - } else -#endif - { - amrex::WriteMultiLevelPlotfile( - plotfilename, finest_level + 1, GetVecOfConstPtrs(mf_plt), plt_VarsName, - Geom(), m_cur_time, istep, refRatio()); - } - -#ifdef PELE_USE_SPRAY - if (do_spray_particles) { - constexpr bool is_spraycheck = false; - for (int lev = 0; lev <= finest_level; ++lev) { - SprayPC->SprayParticleIO(lev, is_spraycheck, plotfilename); - // Remove virtual particles that were made for derived variables - removeVirtualParticles(lev); - } - } -#endif + // Sanity check: ncomp and name list must stay in sync. + // Mirrors PeleC's constructPlotMF AMREX_ASSERT(n_data_items == + // plt_var_names.size()). + AMREX_ASSERT(ncomp == static_cast(a_plt_VarsName.size())); } - void PeleLM::WriteHeader(const std::string& name, const bool is_checkpoint) const {