From df6b6d06365a15acaf6c1f9c1fee4b6d41df2eb5 Mon Sep 17 00:00:00 2001 From: Yoo Date: Fri, 28 Jun 2024 15:08:58 -0700 Subject: [PATCH 01/32] This branch is for visulization of multicut faces/cells. We mainly add a feature to capture the location of multicuts. EB/AMReX_EB2_3D_C.cpp will be the main file to edit since the function 'build_faces' is here. --- Src/EB/AMReX_EB2_3D_C.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 73170adaeeb..c538a101a4e 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -766,6 +766,7 @@ int build_faces (Box const& bx, Array4 const& cell, } }); } else { + //EY: Let's see the location of multicuts before aborting! amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); } } From 734f7aa243e599e0674ad5ff23bf66f8875c3ef7 Mon Sep 17 00:00:00 2001 From: Yoo Date: Fri, 28 Jun 2024 15:32:41 -0700 Subject: [PATCH 02/32] As a baby step, we now can see how many multicuts detected, along with dx, dy, and dz: This is given so that one can consider refining grid to avoid multicuts --- Src/EB/AMReX_EB2_3D_C.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index c538a101a4e..ca4b82ef699 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -767,6 +767,8 @@ int build_faces (Box const& bx, Array4 const& cell, }); } else { //EY: Let's see the location of multicuts before aborting! + amrex::Print() << "dx = " << dx[0] << ", dy = " << dx[1] << ", dz = " << dx[2] << "\n"; + amrex::Print() << "Final *hp = " << *hp << "\n"; amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); } } From fb5c787eec9c5454f90382fb4774c733d5eb9a78 Mon Sep 17 00:00:00 2001 From: Yoo Date: Mon, 1 Jul 2024 17:53:29 -0700 Subject: [PATCH 03/32] All the parts are implemented to write plot files of multicut locations: Please find 'EY' tags for any changes --- Src/EB/AMReX_EB2_3D_C.cpp | 56 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index ca4b82ef699..59e566f735e 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -1,4 +1,7 @@ #include +//EY +#include +#include namespace amrex::EB2 { @@ -460,6 +463,23 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); + // EY: write plot file *just for the locations of multi-cuts------------------------------------ + amrex::MFIter::allowMultipleMFIters(true); + RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, + {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); + Geometry geom(xbx,real_box,CoordSys::cartesian,{0,0,0}); + BoxArray ba(xbx); + DistributionMapping dm(ba); + + MultiFab flag_xbx; + BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); + int nghost = 1; + flag_xbx.define(nodal_ba, dm, 1, nghost); + + std::string m_plot_file{"plt.x."}; + const std::string& plotfilename = amrex::Concatenate(m_plot_file, *hp); + WriteSingleLevelPlotfile(plotfilename, flag_xbx, {"marker_x"}, geom, 0.0, 0); + // Finished plot file ---------------------------------------------------------------------------- } if ((ncuts > 2) || (lym <= small && lyp <= small && lzm <= small && lzp <= small)) { @@ -568,6 +588,23 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); + // EY: write plot file *just for the locations of multi-cuts------------------------------------ + amrex::MFIter::allowMultipleMFIters(true); + RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, + {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); + Geometry geom(ybx,real_box,CoordSys::cartesian,{0,0,0}); + BoxArray ba(ybx); + DistributionMapping dm(ba); + + MultiFab flag_ybx; + BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); + int nghost = 1; + flag_ybx.define(nodal_ba, dm, 1, nghost); + + std::string m_plot_file{"plt.y."}; + const std::string& plotfilename = amrex::Concatenate(m_plot_file, *hp); + WriteSingleLevelPlotfile(plotfilename, flag_ybx, {"marker_y"}, geom, 0.0, 0); + // Finished plot file ---------------------------------------------------------------------------- } if ((ncuts > 2) || (lxm <= small && lxp <= small && lzm <= small && lzp <= small)) { @@ -676,6 +713,23 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); + // EY: write plot file *just for the locations of multi-cuts------------------------------------ + amrex::MFIter::allowMultipleMFIters(true); + RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, + {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); + Geometry geom(zbx,real_box,CoordSys::cartesian,{0,0,0}); + BoxArray ba(zbx); + DistributionMapping dm(ba); + + MultiFab flag_zbx; + BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); + int nghost = 1; + flag_zbx.define(nodal_ba, dm, 1, nghost); + + std::string m_plot_file{"plt.z."}; + const std::string& plotfilename = amrex::Concatenate(m_plot_file, *hp); + WriteSingleLevelPlotfile(plotfilename, flag_zbx, {"marker_z"}, geom, 0.0, 0); + // Finished plot file ---------------------------------------------------------------------------- } if ((ncuts > 2) || (lxm <= small && lxp <= small && lym <= small && lyp <= small)) { @@ -768,7 +822,7 @@ int build_faces (Box const& bx, Array4 const& cell, } else { //EY: Let's see the location of multicuts before aborting! amrex::Print() << "dx = " << dx[0] << ", dy = " << dx[1] << ", dz = " << dx[2] << "\n"; - amrex::Print() << "Final *hp = " << *hp << "\n"; + amrex::Print() << "Total number of multicut cells = " << *hp << "\n"; amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); } } From 70b51119fd6cfcc1b26443e70767a5cb7cb9a6ad Mon Sep 17 00:00:00 2001 From: Yoo Date: Wed, 3 Jul 2024 15:24:55 -0700 Subject: [PATCH 04/32] 1) Added parmparse called "plt_multiple_cuts" which is false by default and you may choose 'true' in your input file if you want to see the multicut locations. -> eb2.plt_multiple_cuts = 'true' 2) If (plt_multiple_cuts), we delay aborting the code until finishing up the loop, including the build_cells part. 3) I just made the int total_multicuts to get the final number of multicuts -- to make sure in case it is not the same number as nmulticuts. --- Src/EB/AMReX_EB2_3D_C.cpp | 20 ++++++++++++++++---- Src/EB/AMReX_EB2_C.H | 5 +++-- Src/EB/AMReX_EB2_Level.H | 12 ++++++++++-- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 59e566f735e..9714f55eda8 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -371,7 +371,8 @@ int build_faces (Box const& bx, Array4 const& cell, Array4 const& m2z, GpuArray const& dx, GpuArray const& problo, - bool cover_multiple_cuts) noexcept + bool cover_multiple_cuts, + bool plt_multiple_cuts) noexcept { Gpu::Buffer nmulticuts = {0}; int* hp = nmulticuts.hostData(); @@ -823,7 +824,13 @@ int build_faces (Box const& bx, Array4 const& cell, //EY: Let's see the location of multicuts before aborting! amrex::Print() << "dx = " << dx[0] << ", dy = " << dx[1] << ", dz = " << dx[2] << "\n"; amrex::Print() << "Total number of multicut cells = " << *hp << "\n"; - amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + + if (plt_multiple_cuts){ + amrex::Print() << "Creating outputs for multicut locations." << "\n"; + } + else{ + amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + } } } @@ -842,7 +849,7 @@ void build_cells (Box const& bx, Array4 const& cell, Array4 const& barea, Array4 const& bcent, Array4 const& bnorm, Array4 const& ctmp, Array4 const& levset, Real small_volfrac, Geometry const& geom, - bool extend_domain_face, bool cover_multiple_cuts, + bool extend_domain_face, bool cover_multiple_cuts, bool plt_multiple_cuts, int& nsmallcells, int& nmulticuts) noexcept { Gpu::Buffer n_smallcell_multicuts = {0,0}; @@ -987,7 +994,12 @@ void build_cells (Box const& bx, Array4 const& cell, if (nsmallcells > 0 || nmulticuts > 0) { if (!cover_multiple_cuts && nmulticuts > 0) { - amrex::Abort("amrex::EB2::build_cells: multi-cuts not supported"); + if (plt_multiple_cuts){ + amrex::Print() << "Passing EB2::build_cells" << "\n"; + } + else{ + amrex::Abort("amrex::EB2::build_cells: multi-cuts not supported"); + } } return; } else { diff --git a/Src/EB/AMReX_EB2_C.H b/Src/EB/AMReX_EB2_C.H index 49db3a270bd..8ed5a41e179 100644 --- a/Src/EB/AMReX_EB2_C.H +++ b/Src/EB/AMReX_EB2_C.H @@ -55,7 +55,8 @@ int build_faces (Box const& bx, Array4 const& cell, Array4 const& m2z, GpuArray const& dx, GpuArray const& problo, - bool cover_multiple_cuts) noexcept; + bool cover_multiple_cuts, + bool plt_multiple_cuts) noexcept; //EY void build_cells (Box const& bx, Array4 const& cell, Array4 const& fx, Array4 const& fy, @@ -69,7 +70,7 @@ void build_cells (Box const& bx, Array4 const& cell, Array4 const& barea, Array4 const& bcent, Array4 const& bnorm, Array4 const& ctmp, Array4 const& levset, Real small_volfrac, Geometry const& geom, - bool extend_domain_face, bool cover_multiple_cuts, + bool extend_domain_face, bool cover_multiple_cuts, bool plt_multiple_cuts, //EY int& nsmallcells, int& nmulticuts) noexcept; void set_connection_flags(Box const& bx, Box const& bxg1, diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 7e30e51a6ba..38f482e9a29 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -167,12 +167,14 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Real small_volfrac = 1.e-14; #endif bool cover_multiple_cuts = false; + bool plt_multiple_cuts = false; //EY int maxiter = 32; { ParmParse pp("eb2"); pp.queryAdd("small_volfrac", small_volfrac); pp.queryAdd("cover_multiple_cuts", cover_multiple_cuts); pp.queryAdd("maxiter", maxiter); + pp.queryAdd("plt_multiple_cuts", plt_multiple_cuts); //EY } maxiter = std::min(100000, maxiter); @@ -315,6 +317,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, amrex::ignore_unused(hybrid); int iter = 0; + int total_multicuts = 0; //EY for (; iter < maxiter; ++iter) { int nsmallcells = 0; @@ -415,7 +418,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, nmc = build_faces(vbx, cfg, ftx, fty, ftz, xdg, ydg, zdg, lst, xip, yip, zip, apx, apy, apz, fcx, fcy, fcz, - xm2, ym2, zm2, dx, problo, cover_multiple_cuts); + xm2, ym2, zm2, dx, problo, cover_multiple_cuts, plt_multiple_cuts); cellflagtmp.resize(m_cellflag[mfi].box()); Elixir cellflagtmp_eli = cellflagtmp.elixir(); @@ -424,7 +427,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, build_cells(vbx, cfg, ftx, fty, ftz, apx, apy, apz, fcx, fcy, fcz, xm2, ym2, zm2, dx, vfr, ctr, bar, bct, bnm, cfgtmp, lst, - small_volfrac, geom, extend_domain_face, cover_multiple_cuts, + small_volfrac, geom, extend_domain_face, cover_multiple_cuts, plt_multiple_cuts, nsm, nmc); // Because it is used in a synchronous reduction kernel in @@ -496,6 +499,11 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, } } } + total_multicuts = total_multicuts + nmulticuts; + } + if (plt_multiple_cuts){ + amrex::Print() << "Grand total multicuts = " << total_multicuts << "\n"; + amrex::Abort("EY: amrex::EB2::build_faces: more than 2 cuts not supported"); // move this } AMREX_ALWAYS_ASSERT_WITH_MESSAGE(iter < maxiter, "EB: failed to fix small cells"); From b35cba5d616a12dae6238c83f2bb0c40f6547f8b Mon Sep 17 00:00:00 2001 From: Yoo Date: Thu, 18 Jul 2024 16:26:22 -0700 Subject: [PATCH 05/32] Edited the parmparse option to turn on/off visualization/printing of multicuts. We can now print out the index of multicut boxes but this current output seems to be no good use. Need to adjust more. --- Src/EB/AMReX_EB2_3D_C.cpp | 117 +++++++++++++++++++++----------------- Src/EB/AMReX_EB2_Level.H | 2 +- 2 files changed, 67 insertions(+), 52 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 9714f55eda8..6b3543b9e5c 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -464,23 +464,28 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); - // EY: write plot file *just for the locations of multi-cuts------------------------------------ - amrex::MFIter::allowMultipleMFIters(true); - RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, - {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); - Geometry geom(xbx,real_box,CoordSys::cartesian,{0,0,0}); - BoxArray ba(xbx); - DistributionMapping dm(ba); - - MultiFab flag_xbx; - BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); - int nghost = 1; - flag_xbx.define(nodal_ba, dm, 1, nghost); - - std::string m_plot_file{"plt.x."}; - const std::string& plotfilename = amrex::Concatenate(m_plot_file, *hp); - WriteSingleLevelPlotfile(plotfilename, flag_xbx, {"marker_x"}, geom, 0.0, 0); - // Finished plot file ---------------------------------------------------------------------------- + if (plt_multiple_cuts){ + // EY: Write txt file to for multicuts index + const std::string& datafilename = amrex::Concatenate("index_multicuts", *hp); + amrex::PrintToFile(datafilename) << "fx(i,j,k) index: (i,j,k) = (" << i << "," << j << "," << k << ") \n"; + // EY: write plot file *just for the locations of multi-cuts------------------------------------ + amrex::MFIter::allowMultipleMFIters(true); + RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, + {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); + Geometry geom(xbx,real_box,CoordSys::cartesian,{0,0,0}); + BoxArray ba(xbx); + DistributionMapping dm(ba); + + MultiFab flag_xbx; + BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); + int nghost = 1; + flag_xbx.define(nodal_ba, dm, 1, nghost); + + // std::string m_plot_file{"plt."}; + const std::string& plotfilename = amrex::Concatenate("plt", *hp); + WriteSingleLevelPlotfile(plotfilename, flag_xbx, {"marker_x"}, geom, 0.0, 0); + // Finished plot file ---------------------------------------------------------------------------- + } } if ((ncuts > 2) || (lym <= small && lyp <= small && lzm <= small && lzp <= small)) { @@ -589,23 +594,28 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); - // EY: write plot file *just for the locations of multi-cuts------------------------------------ - amrex::MFIter::allowMultipleMFIters(true); - RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, - {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); - Geometry geom(ybx,real_box,CoordSys::cartesian,{0,0,0}); - BoxArray ba(ybx); - DistributionMapping dm(ba); - - MultiFab flag_ybx; - BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); - int nghost = 1; - flag_ybx.define(nodal_ba, dm, 1, nghost); - - std::string m_plot_file{"plt.y."}; - const std::string& plotfilename = amrex::Concatenate(m_plot_file, *hp); - WriteSingleLevelPlotfile(plotfilename, flag_ybx, {"marker_y"}, geom, 0.0, 0); - // Finished plot file ---------------------------------------------------------------------------- + if (plt_multiple_cuts){ + // EY: Write txt file to for multicuts index + const std::string& datafilename = amrex::Concatenate("index_multicuts", *hp); + amrex::PrintToFile(datafilename) << "fy(i,j,k) index: (i,j,k) = (" << i << "," << j << "," << k << ") \n"; + // EY: write plot file *just for the locations of multi-cuts------------------------------------ + amrex::MFIter::allowMultipleMFIters(true); + RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, + {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); + Geometry geom(ybx,real_box,CoordSys::cartesian,{0,0,0}); + BoxArray ba(ybx); + DistributionMapping dm(ba); + + MultiFab flag_ybx; + BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); + int nghost = 1; + flag_ybx.define(nodal_ba, dm, 1, nghost); + + // std::string m_plot_file{"plt."}; + const std::string& plotfilename = amrex::Concatenate("plt", *hp); + WriteSingleLevelPlotfile(plotfilename, flag_ybx, {"marker_y"}, geom, 0.0, 0); + // Finished plot file ---------------------------------------------------------------------------- + } } if ((ncuts > 2) || (lxm <= small && lxp <= small && lzm <= small && lzp <= small)) { @@ -714,23 +724,28 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); - // EY: write plot file *just for the locations of multi-cuts------------------------------------ - amrex::MFIter::allowMultipleMFIters(true); - RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, - {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); - Geometry geom(zbx,real_box,CoordSys::cartesian,{0,0,0}); - BoxArray ba(zbx); - DistributionMapping dm(ba); - - MultiFab flag_zbx; - BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); - int nghost = 1; - flag_zbx.define(nodal_ba, dm, 1, nghost); - - std::string m_plot_file{"plt.z."}; - const std::string& plotfilename = amrex::Concatenate(m_plot_file, *hp); - WriteSingleLevelPlotfile(plotfilename, flag_zbx, {"marker_z"}, geom, 0.0, 0); - // Finished plot file ---------------------------------------------------------------------------- + if (plt_multiple_cuts){ + // EY: Write txt file to for multicuts index + const std::string& datafilename = amrex::Concatenate("index_multicuts", *hp); + amrex::PrintToFile(datafilename) << "fz(i,j,k) index: (i,j,k) = (" << i << "," << j << "," << k << ") \n"; + // EY: write plot file *just for the locations of multi-cuts------------------------------------ + amrex::MFIter::allowMultipleMFIters(true); + RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, + {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); + Geometry geom(zbx,real_box,CoordSys::cartesian,{0,0,0}); + BoxArray ba(zbx); + DistributionMapping dm(ba); + + MultiFab flag_zbx; + BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); + int nghost = 1; + flag_zbx.define(nodal_ba, dm, 1, nghost); + + // std::string m_plot_file{"plt"}; + const std::string& plotfilename = amrex::Concatenate("plt", *hp); + WriteSingleLevelPlotfile(plotfilename, flag_zbx, {"marker_z"}, geom, 0.0, 0); + // Finished plot file ---------------------------------------------------------------------------- + } } if ((ncuts > 2) || (lxm <= small && lxp <= small && lym <= small && lyp <= small)) { diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 38f482e9a29..08f3a70769e 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -503,7 +503,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, } if (plt_multiple_cuts){ amrex::Print() << "Grand total multicuts = " << total_multicuts << "\n"; - amrex::Abort("EY: amrex::EB2::build_faces: more than 2 cuts not supported"); // move this + amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); // move this } AMREX_ALWAYS_ASSERT_WITH_MESSAGE(iter < maxiter, "EB: failed to fix small cells"); From 5c7561e8738f02b3cca25f0220511c041f48b058 Mon Sep 17 00:00:00 2001 From: Yoo Date: Fri, 19 Jul 2024 13:35:30 -0700 Subject: [PATCH 06/32] Erased everything for plotting; We now generate one text file loc_multicuts.0 that includes physical location of the multicut cells: The coordinates indicate the cell center. --- Src/EB/AMReX_EB2_3D_C.cpp | 63 ++------------------------------------- 1 file changed, 3 insertions(+), 60 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 6b3543b9e5c..fd2382730a9 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -465,26 +465,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - // EY: Write txt file to for multicuts index - const std::string& datafilename = amrex::Concatenate("index_multicuts", *hp); - amrex::PrintToFile(datafilename) << "fx(i,j,k) index: (i,j,k) = (" << i << "," << j << "," << k << ") \n"; - // EY: write plot file *just for the locations of multi-cuts------------------------------------ - amrex::MFIter::allowMultipleMFIters(true); - RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, - {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); - Geometry geom(xbx,real_box,CoordSys::cartesian,{0,0,0}); - BoxArray ba(xbx); - DistributionMapping dm(ba); - - MultiFab flag_xbx; - BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); - int nghost = 1; - flag_xbx.define(nodal_ba, dm, 1, nghost); - - // std::string m_plot_file{"plt."}; - const std::string& plotfilename = amrex::Concatenate("plt", *hp); - WriteSingleLevelPlotfile(plotfilename, flag_xbx, {"marker_x"}, geom, 0.0, 0); - // Finished plot file ---------------------------------------------------------------------------- + amrex::PrintToFile("loc_multicuts") << "fx: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; } } @@ -595,26 +576,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - // EY: Write txt file to for multicuts index - const std::string& datafilename = amrex::Concatenate("index_multicuts", *hp); - amrex::PrintToFile(datafilename) << "fy(i,j,k) index: (i,j,k) = (" << i << "," << j << "," << k << ") \n"; - // EY: write plot file *just for the locations of multi-cuts------------------------------------ - amrex::MFIter::allowMultipleMFIters(true); - RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, - {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); - Geometry geom(ybx,real_box,CoordSys::cartesian,{0,0,0}); - BoxArray ba(ybx); - DistributionMapping dm(ba); - - MultiFab flag_ybx; - BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); - int nghost = 1; - flag_ybx.define(nodal_ba, dm, 1, nghost); - - // std::string m_plot_file{"plt."}; - const std::string& plotfilename = amrex::Concatenate("plt", *hp); - WriteSingleLevelPlotfile(plotfilename, flag_ybx, {"marker_y"}, geom, 0.0, 0); - // Finished plot file ---------------------------------------------------------------------------- + amrex::PrintToFile("loc_multicuts") << "fy: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; } } @@ -725,26 +687,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - // EY: Write txt file to for multicuts index - const std::string& datafilename = amrex::Concatenate("index_multicuts", *hp); - amrex::PrintToFile(datafilename) << "fz(i,j,k) index: (i,j,k) = (" << i << "," << j << "," << k << ") \n"; - // EY: write plot file *just for the locations of multi-cuts------------------------------------ - amrex::MFIter::allowMultipleMFIters(true); - RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, - {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); - Geometry geom(zbx,real_box,CoordSys::cartesian,{0,0,0}); - BoxArray ba(zbx); - DistributionMapping dm(ba); - - MultiFab flag_zbx; - BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); - int nghost = 1; - flag_zbx.define(nodal_ba, dm, 1, nghost); - - // std::string m_plot_file{"plt"}; - const std::string& plotfilename = amrex::Concatenate("plt", *hp); - WriteSingleLevelPlotfile(plotfilename, flag_zbx, {"marker_z"}, geom, 0.0, 0); - // Finished plot file ---------------------------------------------------------------------------- + amrex::PrintToFile("loc_multicuts") << "fz: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; } } From 92c65facd93ff8676e10c8be393c9df0d451479e Mon Sep 17 00:00:00 2001 From: Yoo Date: Fri, 26 Jul 2024 14:46:49 -0700 Subject: [PATCH 07/32] We found a 'right' place to produce the plot file in AMReX_EB2_Level.H. Next step would be to create a multifab to store locations of the multicut in build_faces function and bring it out to AMReX_EB2_Leve.H. Also, try to output one plot file. --- Src/EB/AMReX_EB2_3D_C.cpp | 3 +++ Src/EB/AMReX_EB2_Level.H | 28 ++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index fd2382730a9..17f265696c6 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -465,6 +465,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ + // Not GPU friendly amrex::PrintToFile("loc_multicuts") << "fx: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; } } @@ -576,6 +577,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ + // Not GPU friendly amrex::PrintToFile("loc_multicuts") << "fy: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; } } @@ -687,6 +689,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ + // Not GPU friendly amrex::PrintToFile("loc_multicuts") << "fz: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; } } diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 08f3a70769e..19a1a49805f 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -15,6 +15,9 @@ #include #include +//EY: Plotting +#include + #ifdef AMREX_USE_OMP #include #endif @@ -479,6 +482,27 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, nsmallcells += nsm; nmulticuts += nmc; } + // EY: Here -> make plot file + if (plt_multiple_cuts){ + amrex::Print() << "Nmulticuts = " << nmulticuts << "\n"; + // Output: plotfiles + int i = 1; int j = 1; int k = 1; + RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, + {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); + + Geometry geom(domain, real_box, CoordSys::cartesian,{0,0,0}); + BoxArray ba(domain); + BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); + DistributionMapping dm(ba); + + int nghost = 1; + MultiFab flag_xbx; + flag_xbx.define(nodal_ba, dm, 1, nghost); + const std::string& plotfilename = amrex::Concatenate("plt", total_multicuts); + WriteSingleLevelPlotfile(plotfilename, flag_xbx, {"marker_x"}, geom, 0.0, 0); + + amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); // move this + } } ParallelAllReduce::Sum({nsmallcells,nmulticuts}, ParallelContext::CommunicatorSub()); @@ -501,10 +525,6 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, } total_multicuts = total_multicuts + nmulticuts; } - if (plt_multiple_cuts){ - amrex::Print() << "Grand total multicuts = " << total_multicuts << "\n"; - amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); // move this - } AMREX_ALWAYS_ASSERT_WITH_MESSAGE(iter < maxiter, "EB: failed to fix small cells"); From 4b4b8904a76cad29be3af2a1afb2b3ded18aba2c Mon Sep 17 00:00:00 2001 From: Yoo Date: Mon, 29 Jul 2024 08:48:23 -0700 Subject: [PATCH 08/32] 1) Created a multicut called 'multicut_bx'. 2) This is used as an input for build_faces. 3) Simply set value - multicut_arr(i,j,k) = 10.0 - whenever we detect multicut. 4) In AMReX_EB2_Level.H (line 494 - ), we first check if multicut_arr stored the multicut locations (value 10.0). - this will be removed once we get right plot file. 5) The hope is that we get "volume" of multicut location when we read the plt file. 6) We probably do not need anything in this line, except the line 508 (WriteSingleLevelPlotfile). - The question is: Is this enought to see the multicuts? --- Src/EB/AMReX_EB2_3D_C.cpp | 6 +++++- Src/EB/AMReX_EB2_C.H | 2 +- Src/EB/AMReX_EB2_Level.H | 42 ++++++++++++++++++++++----------------- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 17f265696c6..6c9c170149c 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -372,7 +372,7 @@ int build_faces (Box const& bx, Array4 const& cell, GpuArray const& dx, GpuArray const& problo, bool cover_multiple_cuts, - bool plt_multiple_cuts) noexcept + bool plt_multiple_cuts, Array4 const& multicut_arr) noexcept { Gpu::Buffer nmulticuts = {0}; int* hp = nmulticuts.hostData(); @@ -465,6 +465,8 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ + multicut_arr(i,j,k) = 10.0; + amrex::Print() << "Multicut array (i,j,k) = (" << i << ", " << j << ", " << k << ") " << "\n"; // Not GPU friendly amrex::PrintToFile("loc_multicuts") << "fx: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; } @@ -577,6 +579,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ + multicut_arr(i,j,k) = 10.0; // Not GPU friendly amrex::PrintToFile("loc_multicuts") << "fy: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; } @@ -689,6 +692,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ + multicut_arr(i,j,k) = 10.0; // Not GPU friendly amrex::PrintToFile("loc_multicuts") << "fz: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; } diff --git a/Src/EB/AMReX_EB2_C.H b/Src/EB/AMReX_EB2_C.H index 8ed5a41e179..fecea96ae09 100644 --- a/Src/EB/AMReX_EB2_C.H +++ b/Src/EB/AMReX_EB2_C.H @@ -56,7 +56,7 @@ int build_faces (Box const& bx, Array4 const& cell, GpuArray const& dx, GpuArray const& problo, bool cover_multiple_cuts, - bool plt_multiple_cuts) noexcept; //EY + bool plt_multiple_cuts, Array4 const& multicut_arr) noexcept; //EY void build_cells (Box const& bx, Array4 const& cell, Array4 const& fx, Array4 const& fy, diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 19a1a49805f..5a78d983c64 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -101,6 +101,9 @@ protected: bool m_has_eb_info = true; IndexSpace const* m_parent; + //EY + MultiFab multicut_bx; + private: template friend class GShopLevel; @@ -295,6 +298,9 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, m_bndryarea.define(m_grids, m_dmap, 1, ng, mf_info); m_bndrycent.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); m_bndrynorm.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); + //EY + multicut_bx.define(m_grids, m_dmap, 1, ng, mf_info); + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_areafrac[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, 1, ng, mf_info); @@ -360,6 +366,9 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Array4 const& bar = m_bndryarea.array(mfi); Array4 const& bct = m_bndrycent.array(mfi); Array4 const& bnm = m_bndrynorm.array(mfi); + //EY: + Array4 const& multicut_arr = multicut_bx.array(mfi); + AMREX_D_TERM(Array4 const& apx = m_areafrac[0].array(mfi);, Array4 const& apy = m_areafrac[1].array(mfi);, Array4 const& apz = m_areafrac[2].array(mfi);); @@ -421,7 +430,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, nmc = build_faces(vbx, cfg, ftx, fty, ftz, xdg, ydg, zdg, lst, xip, yip, zip, apx, apy, apz, fcx, fcy, fcz, - xm2, ym2, zm2, dx, problo, cover_multiple_cuts, plt_multiple_cuts); + xm2, ym2, zm2, dx, problo, cover_multiple_cuts, plt_multiple_cuts, multicut_arr); cellflagtmp.resize(m_cellflag[mfi].box()); Elixir cellflagtmp_eli = cellflagtmp.elixir(); @@ -484,24 +493,21 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, } // EY: Here -> make plot file if (plt_multiple_cuts){ - amrex::Print() << "Nmulticuts = " << nmulticuts << "\n"; + for (MFIter mfi(m_mgf); mfi.isValid(); ++mfi) + { + AMREX_HOST_DEVICE_FOR_3D ( domain_grown, i, j, k, + { + Array4 const& multicut_arr = multicut_bx.array(mfi); + if (multicut_arr(i,j,k) > 1) // Need to fix this condition + { + amrex::Print() << "Multicut array = (" << i << ", " << j << ", " << k << ") \n"; + } + }); + } // Output: plotfiles - int i = 1; int j = 1; int k = 1; - RealBox real_box({AMREX_D_DECL(problo[0]+(i)*dx[0], problo[1]+(j)*dx[1], problo[2]+(k)*dx[2])}, - {AMREX_D_DECL(problo[0]+(i+1)*dx[0], problo[1]+(j+1)*dx[1], problo[2]+(k+1)*dx[2])}); - - Geometry geom(domain, real_box, CoordSys::cartesian,{0,0,0}); - BoxArray ba(domain); - BoxArray nodal_ba = amrex::convert(ba, IntVect::TheNodeVector()); - DistributionMapping dm(ba); - - int nghost = 1; - MultiFab flag_xbx; - flag_xbx.define(nodal_ba, dm, 1, nghost); - const std::string& plotfilename = amrex::Concatenate("plt", total_multicuts); - WriteSingleLevelPlotfile(plotfilename, flag_xbx, {"marker_x"}, geom, 0.0, 0); - - amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); // move this + WriteSingleLevelPlotfile("plt", multicut_bx, {"multicut"}, geom, 0.0, 0); + + amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); } } From 823e97f689b540a6d61a591f1c9ed124c46fc568 Mon Sep 17 00:00:00 2001 From: Yoo Date: Mon, 29 Jul 2024 09:01:19 -0700 Subject: [PATCH 09/32] Deleted non-necessary printings. --- Src/EB/AMReX_EB2_3D_C.cpp | 1 - Src/EB/AMReX_EB2_Level.H | 16 ++-------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 6c9c170149c..5fd4bdab9de 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -466,7 +466,6 @@ int build_faces (Box const& bx, Array4 const& cell, Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ multicut_arr(i,j,k) = 10.0; - amrex::Print() << "Multicut array (i,j,k) = (" << i << ", " << j << ", " << k << ") " << "\n"; // Not GPU friendly amrex::PrintToFile("loc_multicuts") << "fx: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; } diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 5a78d983c64..9047fc9fb55 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -492,21 +492,9 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, nmulticuts += nmc; } // EY: Here -> make plot file - if (plt_multiple_cuts){ - for (MFIter mfi(m_mgf); mfi.isValid(); ++mfi) - { - AMREX_HOST_DEVICE_FOR_3D ( domain_grown, i, j, k, - { - Array4 const& multicut_arr = multicut_bx.array(mfi); - if (multicut_arr(i,j,k) > 1) // Need to fix this condition - { - amrex::Print() << "Multicut array = (" << i << ", " << j << ", " << k << ") \n"; - } - }); - } - // Output: plotfiles + if (plt_multiple_cuts) + { WriteSingleLevelPlotfile("plt", multicut_bx, {"multicut"}, geom, 0.0, 0); - amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); } } From 4007d81c838b5f483bab97758f230cddfb67b07c Mon Sep 17 00:00:00 2001 From: Yoo Date: Mon, 29 Jul 2024 09:03:52 -0700 Subject: [PATCH 10/32] Deleted total_multicuts value --- Src/EB/AMReX_EB2_Level.H | 2 -- 1 file changed, 2 deletions(-) diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 9047fc9fb55..75eb25571f2 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -326,7 +326,6 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, amrex::ignore_unused(hybrid); int iter = 0; - int total_multicuts = 0; //EY for (; iter < maxiter; ++iter) { int nsmallcells = 0; @@ -517,7 +516,6 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, } } } - total_multicuts = total_multicuts + nmulticuts; } AMREX_ALWAYS_ASSERT_WITH_MESSAGE(iter < maxiter, "EB: failed to fix small cells"); From 8e676505c8ac570aa23c7b206be8bf9c3648a939 Mon Sep 17 00:00:00 2001 From: Yoo Date: Wed, 7 Aug 2024 14:54:25 -0700 Subject: [PATCH 11/32] minor changed after Bruce's comments. Run a tutorial input with a rotor stl on Kestrel. --- Src/EB/AMReX_EB2_3D_C.cpp | 25 +++++++++++++++---------- Src/EB/AMReX_EB2_Level.H | 8 ++++---- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 5fd4bdab9de..1f47f77e919 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -465,9 +465,11 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - multicut_arr(i,j,k) = 10.0; - // Not GPU friendly - amrex::PrintToFile("loc_multicuts") << "fx: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; + multicut_arr(i,j,k) = ncuts; + + #ifdef AMREX_USE_GPU + amrex::PrintToFile("loc_multicuts") << "fx: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; + #endif } } @@ -578,9 +580,11 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - multicut_arr(i,j,k) = 10.0; - // Not GPU friendly - amrex::PrintToFile("loc_multicuts") << "fy: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; + multicut_arr(i,j,k) = ncuts; + + #ifdef AMREX_USE_GPU + amrex::PrintToFile("loc_multicuts") << "fy: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; + #endif } } @@ -691,9 +695,11 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - multicut_arr(i,j,k) = 10.0; - // Not GPU friendly - amrex::PrintToFile("loc_multicuts") << "fz: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; + multicut_arr(i,j,k) = ncuts; + + #ifdef AMREX_USE_GPU + amrex::PrintToFile("loc_multicuts") << "fz: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; + #endif } } @@ -786,7 +792,6 @@ int build_faces (Box const& bx, Array4 const& cell, }); } else { //EY: Let's see the location of multicuts before aborting! - amrex::Print() << "dx = " << dx[0] << ", dy = " << dx[1] << ", dz = " << dx[2] << "\n"; amrex::Print() << "Total number of multicut cells = " << *hp << "\n"; if (plt_multiple_cuts){ diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 75eb25571f2..3a1af1ed8df 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -102,7 +102,7 @@ protected: IndexSpace const* m_parent; //EY - MultiFab multicut_bx; + MultiFab multicut_count; private: template friend class GShopLevel; @@ -299,7 +299,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, m_bndrycent.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); m_bndrynorm.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); //EY - multicut_bx.define(m_grids, m_dmap, 1, ng, mf_info); + multicut_count.define(m_grids, m_dmap, 1, ng, mf_info); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_areafrac[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), @@ -366,7 +366,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Array4 const& bct = m_bndrycent.array(mfi); Array4 const& bnm = m_bndrynorm.array(mfi); //EY: - Array4 const& multicut_arr = multicut_bx.array(mfi); + Array4 const& multicut_arr = multicut_count.array(mfi); AMREX_D_TERM(Array4 const& apx = m_areafrac[0].array(mfi);, Array4 const& apy = m_areafrac[1].array(mfi);, @@ -493,7 +493,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, // EY: Here -> make plot file if (plt_multiple_cuts) { - WriteSingleLevelPlotfile("plt", multicut_bx, {"multicut"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt", multicut_count, {"multicut"}, geom, 0.0, 0); amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); } } From 1a7edb343f81710d49587e13b648dd3f76e80aa3 Mon Sep 17 00:00:00 2001 From: Yoo Date: Thu, 15 Aug 2024 15:39:18 -0700 Subject: [PATCH 12/32] We now create separate plot files containing multi-cut locations from xbx, ybx, and zbx: next step will be plotting face-center multifabs since they are multicut faces we are plotting now. --- Src/EB/AMReX_EB2_3D_C.cpp | 16 ++++++------ Src/EB/AMReX_EB2_C.H | 2 +- Src/EB/AMReX_EB2_Level.H | 53 +++++++++++++++++++++++++++++++-------- 3 files changed, 51 insertions(+), 20 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 1f47f77e919..76fec00037f 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -372,7 +372,7 @@ int build_faces (Box const& bx, Array4 const& cell, GpuArray const& dx, GpuArray const& problo, bool cover_multiple_cuts, - bool plt_multiple_cuts, Array4 const& multicut_arr) noexcept + bool plt_multiple_cuts, Array4 const& mt_fcx, Array4 const& mt_fcy, Array4 const& mt_fcz) noexcept { Gpu::Buffer nmulticuts = {0}; int* hp = nmulticuts.hostData(); @@ -388,7 +388,7 @@ int build_faces (Box const& bx, Array4 const& cell, const Real dzinv = 1.0_rt/dx[2]; const Box& xbx = amrex::grow(amrex::surroundingNodes(bx,0),1); - AMREX_HOST_DEVICE_FOR_3D ( xbx, i, j, k, + AMREX_HOST_DEVICE_FOR_3D ( xbx, i, j, k, // this is face index { if (fx(i,j,k) == Type::regular) { apx(i,j,k) = 1.0_rt; @@ -465,9 +465,9 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - multicut_arr(i,j,k) = ncuts; + mt_fcx(i,j,k,0) = 10.0; - #ifdef AMREX_USE_GPU + #ifndef AMREX_USE_GPU amrex::PrintToFile("loc_multicuts") << "fx: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; #endif } @@ -580,9 +580,9 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - multicut_arr(i,j,k) = ncuts; + mt_fcy(i,j,k,0) = 10.0; - #ifdef AMREX_USE_GPU + #ifndef AMREX_USE_GPU amrex::PrintToFile("loc_multicuts") << "fy: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; #endif } @@ -695,9 +695,9 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - multicut_arr(i,j,k) = ncuts; + mt_fcz(i,j,k,0) = 10.0; - #ifdef AMREX_USE_GPU + #ifndef AMREX_USE_GPU amrex::PrintToFile("loc_multicuts") << "fz: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; #endif } diff --git a/Src/EB/AMReX_EB2_C.H b/Src/EB/AMReX_EB2_C.H index fecea96ae09..9afc045e265 100644 --- a/Src/EB/AMReX_EB2_C.H +++ b/Src/EB/AMReX_EB2_C.H @@ -56,7 +56,7 @@ int build_faces (Box const& bx, Array4 const& cell, GpuArray const& dx, GpuArray const& problo, bool cover_multiple_cuts, - bool plt_multiple_cuts, Array4 const& multicut_arr) noexcept; //EY + bool plt_multiple_cuts, Array4 const& mt_fcx, Array4 const& mt_fcy, Array4 const& mt_fcz) noexcept; //EY void build_cells (Box const& bx, Array4 const& cell, Array4 const& fx, Array4 const& fy, diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 3a1af1ed8df..f372eb91803 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -102,7 +102,12 @@ protected: IndexSpace const* m_parent; //EY - MultiFab multicut_count; + Vector multicut_mf; // test in progress + Array multicut_face; // store on face-centered + + MultiFab multicut_fcx; + MultiFab multicut_fcy; + MultiFab multicut_fcz; private: template friend class GShopLevel; @@ -299,13 +304,23 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, m_bndrycent.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); m_bndrynorm.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); //EY - multicut_count.define(m_grids, m_dmap, 1, ng, mf_info); + // amrex::Print() << "MAKE VECTOR MULTIFAB MF" << "\n"; + // multicut_mf[0].define(m_grids, m_dmap, 1, 0, mf_info, MultiCutFab()); // this should be a leveldata + // amrex::Print() << "AFTER VECTOR MULTIFAB MF" << "\n"; + + multicut_fcx.define(m_grids, m_dmap, 1, ng, mf_info); //this is cell-centered values storage + multicut_fcy.define(m_grids, m_dmap, 1, ng, mf_info); // need to be fora similar to m_areafrac + multicut_fcz.define(m_grids, m_dmap, 1, ng, mf_info); // should be a face-centered boxes for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_areafrac[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, 1, ng, mf_info); m_facecent[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, AMREX_SPACEDIM-1, ng, mf_info); + // EY: multicuts------------------------------------------------------------------------ + multicut_face[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), + m_dmap, AMREX_SPACEDIM-1, ng, mf_info); + //-------------------------------------------------------------------------------------- IntVect edge_type{1}; edge_type[idim] = 0; m_edgecent[idim].define(amrex::convert(m_grids, edge_type), m_dmap, 1, ng, mf_info); } @@ -365,8 +380,16 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Array4 const& bar = m_bndryarea.array(mfi); Array4 const& bct = m_bndrycent.array(mfi); Array4 const& bnm = m_bndrynorm.array(mfi); - //EY: - Array4 const& multicut_arr = multicut_count.array(mfi); + //EY: multicuts------------------------------------------------------- + //Individual MFs + Array4 const& multicut_arr_x = multicut_fcx.array(mfi); + Array4 const& multicut_arr_y = multicut_fcy.array(mfi); + Array4 const& multicut_arr_z = multicut_fcz.array(mfi); + //face center MF + AMREX_D_TERM(Array4 const& mt_fcx = multicut_face[0].array(mfi);, + Array4 const& mt_fcy = multicut_face[1].array(mfi);, + Array4 const& mt_fcz = multicut_face[2].array(mfi);); + //---------------------------------------------------------------------- AMREX_D_TERM(Array4 const& apx = m_areafrac[0].array(mfi);, Array4 const& apy = m_areafrac[1].array(mfi);, @@ -429,7 +452,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, nmc = build_faces(vbx, cfg, ftx, fty, ftz, xdg, ydg, zdg, lst, xip, yip, zip, apx, apy, apz, fcx, fcy, fcz, - xm2, ym2, zm2, dx, problo, cover_multiple_cuts, plt_multiple_cuts, multicut_arr); + xm2, ym2, zm2, dx, problo, cover_multiple_cuts, plt_multiple_cuts, multicut_arr_x, multicut_arr_y, multicut_arr_z); cellflagtmp.resize(m_cellflag[mfi].box()); Elixir cellflagtmp_eli = cellflagtmp.elixir(); @@ -490,15 +513,23 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, nsmallcells += nsm; nmulticuts += nmc; } - // EY: Here -> make plot file - if (plt_multiple_cuts) - { - WriteSingleLevelPlotfile("plt", multicut_count, {"multicut"}, geom, 0.0, 0); - amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); - } } ParallelAllReduce::Sum({nsmallcells,nmulticuts}, ParallelContext::CommunicatorSub()); + // EY: Here -> make plot file + if (plt_multiple_cuts) + { + int finest_level = 0; + int lev = finest_level; + amrex::Print() << "MAKE PLOT FILE FOR MULTICUTS" << "\n"; + + WriteSingleLevelPlotfile("plt.x", multicut_fcx, {"multicut_x"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.y", multicut_fcy, {"multicut_y"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.z", multicut_fcz, {"multicut_z"}, geom, 0.0, 0); + + amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + } + if (nsmallcells == 0 && nmulticuts == 0) { break; } else { From 5fe3c8ecb6640649951e42102bbad4835ff869af Mon Sep 17 00:00:00 2001 From: Yoo Date: Mon, 26 Aug 2024 17:39:50 -0600 Subject: [PATCH 13/32] cleaned up some useless comments. Now writing (i,j,k) and face boxes locations in using PrintToFile in AMReX_EB2_3D_C.cpp file. Also, most importantly, I fixed a stupid if-condition mistake in AMReX_EB2_Level.H: line 502. We should not abort the code when no multicuts present but plt_multiple_cuts = true --- Src/EB/AMReX_EB2_3D_C.cpp | 20 +++++++++++++------- Src/EB/AMReX_EB2_C.H | 4 ++-- Src/EB/AMReX_EB2_Level.H | 29 +++++------------------------ 3 files changed, 20 insertions(+), 33 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 76fec00037f..ff81ba613fc 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -1,5 +1,5 @@ #include -//EY + #include #include @@ -468,7 +468,9 @@ int build_faces (Box const& bx, Array4 const& cell, mt_fcx(i,j,k,0) = 10.0; #ifndef AMREX_USE_GPU - amrex::PrintToFile("loc_multicuts") << "fx: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; + amrex::PrintToFile("loc_multicuts") << "xbx = " << xbx << "\n"; + amrex::PrintToFile("loc_multicuts") << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; #endif } } @@ -583,7 +585,9 @@ int build_faces (Box const& bx, Array4 const& cell, mt_fcy(i,j,k,0) = 10.0; #ifndef AMREX_USE_GPU - amrex::PrintToFile("loc_multicuts") << "fy: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; + amrex::PrintToFile("loc_multicuts") << "ybx = " << ybx << "\n"; + amrex::PrintToFile("loc_multicuts") << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; #endif } } @@ -698,7 +702,9 @@ int build_faces (Box const& bx, Array4 const& cell, mt_fcz(i,j,k,0) = 10.0; #ifndef AMREX_USE_GPU - amrex::PrintToFile("loc_multicuts") << "fz: (x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; + amrex::PrintToFile("loc_multicuts") << "zbx = " << zbx << "\n"; + amrex::PrintToFile("loc_multicuts") << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; #endif } } @@ -791,14 +797,14 @@ int build_faces (Box const& bx, Array4 const& cell, } }); } else { - //EY: Let's see the location of multicuts before aborting! + // Passing now to plot the location of multicuts before aborting! amrex::Print() << "Total number of multicut cells = " << *hp << "\n"; if (plt_multiple_cuts){ - amrex::Print() << "Creating outputs for multicut locations." << "\n"; + amrex::Print() << "Creating outputs for multicut locations..." << "\n"; } else{ - amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported!"); } } } diff --git a/Src/EB/AMReX_EB2_C.H b/Src/EB/AMReX_EB2_C.H index 9afc045e265..3f31d883574 100644 --- a/Src/EB/AMReX_EB2_C.H +++ b/Src/EB/AMReX_EB2_C.H @@ -56,7 +56,7 @@ int build_faces (Box const& bx, Array4 const& cell, GpuArray const& dx, GpuArray const& problo, bool cover_multiple_cuts, - bool plt_multiple_cuts, Array4 const& mt_fcx, Array4 const& mt_fcy, Array4 const& mt_fcz) noexcept; //EY + bool plt_multiple_cuts, Array4 const& mt_fcx, Array4 const& mt_fcy, Array4 const& mt_fcz) noexcept; void build_cells (Box const& bx, Array4 const& cell, Array4 const& fx, Array4 const& fy, @@ -70,7 +70,7 @@ void build_cells (Box const& bx, Array4 const& cell, Array4 const& barea, Array4 const& bcent, Array4 const& bnorm, Array4 const& ctmp, Array4 const& levset, Real small_volfrac, Geometry const& geom, - bool extend_domain_face, bool cover_multiple_cuts, bool plt_multiple_cuts, //EY + bool extend_domain_face, bool cover_multiple_cuts, bool plt_multiple_cuts, int& nsmallcells, int& nmulticuts) noexcept; void set_connection_flags(Box const& bx, Box const& bxg1, diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index f372eb91803..35de5645a91 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -101,10 +101,6 @@ protected: bool m_has_eb_info = true; IndexSpace const* m_parent; - //EY - Vector multicut_mf; // test in progress - Array multicut_face; // store on face-centered - MultiFab multicut_fcx; MultiFab multicut_fcy; MultiFab multicut_fcz; @@ -303,10 +299,6 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, m_bndryarea.define(m_grids, m_dmap, 1, ng, mf_info); m_bndrycent.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); m_bndrynorm.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); - //EY - // amrex::Print() << "MAKE VECTOR MULTIFAB MF" << "\n"; - // multicut_mf[0].define(m_grids, m_dmap, 1, 0, mf_info, MultiCutFab()); // this should be a leveldata - // amrex::Print() << "AFTER VECTOR MULTIFAB MF" << "\n"; multicut_fcx.define(m_grids, m_dmap, 1, ng, mf_info); //this is cell-centered values storage multicut_fcy.define(m_grids, m_dmap, 1, ng, mf_info); // need to be fora similar to m_areafrac @@ -317,10 +309,6 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, m_dmap, 1, ng, mf_info); m_facecent[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, AMREX_SPACEDIM-1, ng, mf_info); - // EY: multicuts------------------------------------------------------------------------ - multicut_face[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), - m_dmap, AMREX_SPACEDIM-1, ng, mf_info); - //-------------------------------------------------------------------------------------- IntVect edge_type{1}; edge_type[idim] = 0; m_edgecent[idim].define(amrex::convert(m_grids, edge_type), m_dmap, 1, ng, mf_info); } @@ -380,16 +368,10 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Array4 const& bar = m_bndryarea.array(mfi); Array4 const& bct = m_bndrycent.array(mfi); Array4 const& bnm = m_bndrynorm.array(mfi); - //EY: multicuts------------------------------------------------------- - //Individual MFs + Array4 const& multicut_arr_x = multicut_fcx.array(mfi); Array4 const& multicut_arr_y = multicut_fcy.array(mfi); Array4 const& multicut_arr_z = multicut_fcz.array(mfi); - //face center MF - AMREX_D_TERM(Array4 const& mt_fcx = multicut_face[0].array(mfi);, - Array4 const& mt_fcy = multicut_face[1].array(mfi);, - Array4 const& mt_fcz = multicut_face[2].array(mfi);); - //---------------------------------------------------------------------- AMREX_D_TERM(Array4 const& apx = m_areafrac[0].array(mfi);, Array4 const& apy = m_areafrac[1].array(mfi);, @@ -516,12 +498,11 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, } ParallelAllReduce::Sum({nsmallcells,nmulticuts}, ParallelContext::CommunicatorSub()); - // EY: Here -> make plot file - if (plt_multiple_cuts) + + if (plt_multiple_cuts && nmulticuts > 0) { - int finest_level = 0; - int lev = finest_level; - amrex::Print() << "MAKE PLOT FILE FOR MULTICUTS" << "\n"; + amrex::Print() << "Total number of multicuts = " << nmulticuts << "\n"; + amrex::Print() << "plotting multicut locations..." << "\n"; WriteSingleLevelPlotfile("plt.x", multicut_fcx, {"multicut_x"}, geom, 0.0, 0); WriteSingleLevelPlotfile("plt.y", multicut_fcy, {"multicut_y"}, geom, 0.0, 0); From 2fd9f4d4d5c3dbc95b74a13e78d97ad53f6786fb Mon Sep 17 00:00:00 2001 From: Yoo Date: Thu, 15 Aug 2024 15:39:18 -0700 Subject: [PATCH 14/32] We now create separate plot files containing multi-cut locations from xbx, ybx, and zbx: next step will be plotting face-center multifabs since they are multicut faces we are plotting now. --- Src/EB/AMReX_EB2_Level.H | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 35de5645a91..7e1692aed96 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -309,6 +309,10 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, m_dmap, 1, ng, mf_info); m_facecent[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, AMREX_SPACEDIM-1, ng, mf_info); + // EY: multicuts------------------------------------------------------------------------ + multicut_face[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), + m_dmap, AMREX_SPACEDIM-1, ng, mf_info); + //-------------------------------------------------------------------------------------- IntVect edge_type{1}; edge_type[idim] = 0; m_edgecent[idim].define(amrex::convert(m_grids, edge_type), m_dmap, 1, ng, mf_info); } From 1b1a970a9f7e1ebe31362299ca07436cd21e3e96 Mon Sep 17 00:00:00 2001 From: Yoo Date: Tue, 27 Aug 2024 14:57:31 -0600 Subject: [PATCH 15/32] Cleanup more no useful comments && using now std::ofstream for text file with data multicuts. See AMReX_EB2_3D_C.cpp --- Src/EB/AMReX_EB2_3D_C.cpp | 43 +++++++++++++++++++++++++++++---------- Src/EB/AMReX_EB2_Level.H | 15 +++++--------- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index ff81ba613fc..c9d4b47491d 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -377,6 +377,10 @@ int build_faces (Box const& bx, Array4 const& cell, Gpu::Buffer nmulticuts = {0}; int* hp = nmulticuts.hostData(); int* dp = nmulticuts.data(); + // delete the previous dat_multicuts.txt file + std::ofstream ofs; + ofs.open("dat_multicuts.txt", std::ofstream::out | std::ofstream::trunc); + ofs.close(); #ifdef AMREX_USE_FLOAT constexpr Real small = 1.e-5_rt; @@ -468,9 +472,16 @@ int build_faces (Box const& bx, Array4 const& cell, mt_fcx(i,j,k,0) = 10.0; #ifndef AMREX_USE_GPU - amrex::PrintToFile("loc_multicuts") << "xbx = " << xbx << "\n"; - amrex::PrintToFile("loc_multicuts") << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; + std::ofstream fout("dat_multicuts.txt", std::ios::app); + fout << "xbx = " << xbx << std::endl; + fout << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + << "(x,y,z) = (" << + problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" + << std::endl; + + // amrex::PrintToFile("loc_multicuts") << "xbx = " << xbx << "\n"; + // amrex::PrintToFile("loc_multicuts") << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + // << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; #endif } } @@ -585,9 +596,15 @@ int build_faces (Box const& bx, Array4 const& cell, mt_fcy(i,j,k,0) = 10.0; #ifndef AMREX_USE_GPU - amrex::PrintToFile("loc_multicuts") << "ybx = " << ybx << "\n"; - amrex::PrintToFile("loc_multicuts") << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; + std::ofstream fout("dat_multicuts.txt", std::ios::app); + fout << "ybx = " << ybx << std::endl; + fout << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + << "(x,y,z) = (" << + problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" + << std::endl; + // amrex::PrintToFile("loc_multicuts") << "ybx = " << ybx << "\n"; + // amrex::PrintToFile("loc_multicuts") << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + // << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; #endif } } @@ -702,9 +719,15 @@ int build_faces (Box const& bx, Array4 const& cell, mt_fcz(i,j,k,0) = 10.0; #ifndef AMREX_USE_GPU - amrex::PrintToFile("loc_multicuts") << "zbx = " << zbx << "\n"; - amrex::PrintToFile("loc_multicuts") << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; + std::ofstream fout("dat_multicuts.txt", std::ios::app); + fout << "zbx = " << zbx << std::endl; + fout << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + << "(x,y,z) = (" << + problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" + << std::endl; + // amrex::PrintToFile("loc_multicuts") << "zbx = " << zbx << "\n"; + // amrex::PrintToFile("loc_multicuts") << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + // << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; #endif } } @@ -798,8 +821,6 @@ int build_faces (Box const& bx, Array4 const& cell, }); } else { // Passing now to plot the location of multicuts before aborting! - amrex::Print() << "Total number of multicut cells = " << *hp << "\n"; - if (plt_multiple_cuts){ amrex::Print() << "Creating outputs for multicut locations..." << "\n"; } diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 7e1692aed96..e78515caaaa 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -15,7 +15,6 @@ #include #include -//EY: Plotting #include #ifdef AMREX_USE_OMP @@ -174,14 +173,14 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Real small_volfrac = 1.e-14; #endif bool cover_multiple_cuts = false; - bool plt_multiple_cuts = false; //EY + bool plt_multiple_cuts = false; int maxiter = 32; { ParmParse pp("eb2"); pp.queryAdd("small_volfrac", small_volfrac); pp.queryAdd("cover_multiple_cuts", cover_multiple_cuts); pp.queryAdd("maxiter", maxiter); - pp.queryAdd("plt_multiple_cuts", plt_multiple_cuts); //EY + pp.queryAdd("plt_multiple_cuts", plt_multiple_cuts); } maxiter = std::min(100000, maxiter); @@ -300,19 +299,15 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, m_bndrycent.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); m_bndrynorm.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); - multicut_fcx.define(m_grids, m_dmap, 1, ng, mf_info); //this is cell-centered values storage - multicut_fcy.define(m_grids, m_dmap, 1, ng, mf_info); // need to be fora similar to m_areafrac - multicut_fcz.define(m_grids, m_dmap, 1, ng, mf_info); // should be a face-centered boxes + multicut_fcx.define(m_grids, m_dmap, 1, ng, mf_info); + multicut_fcy.define(m_grids, m_dmap, 1, ng, mf_info); + multicut_fcz.define(m_grids, m_dmap, 1, ng, mf_info); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_areafrac[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, 1, ng, mf_info); m_facecent[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, AMREX_SPACEDIM-1, ng, mf_info); - // EY: multicuts------------------------------------------------------------------------ - multicut_face[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), - m_dmap, AMREX_SPACEDIM-1, ng, mf_info); - //-------------------------------------------------------------------------------------- IntVect edge_type{1}; edge_type[idim] = 0; m_edgecent[idim].define(amrex::convert(m_grids, edge_type), m_dmap, 1, ng, mf_info); } From cc0af83ee46d6f58bb6c4385b71138aac9bcf307 Mon Sep 17 00:00:00 2001 From: Yoo Date: Wed, 28 Aug 2024 15:00:53 -0600 Subject: [PATCH 16/32] Removed #ifndef condition for GPU use since we no longer use amrex::PrintToFile, it seems like we don't need this condition for GPU --- Src/EB/AMReX_EB2_3D_C.cpp | 52 ++++++++++++++------------------------- 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index c9d4b47491d..6dbed6bf6cc 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -471,18 +471,12 @@ int build_faces (Box const& bx, Array4 const& cell, if (plt_multiple_cuts){ mt_fcx(i,j,k,0) = 10.0; - #ifndef AMREX_USE_GPU - std::ofstream fout("dat_multicuts.txt", std::ios::app); - fout << "xbx = " << xbx << std::endl; - fout << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - << "(x,y,z) = (" << - problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" - << std::endl; - - // amrex::PrintToFile("loc_multicuts") << "xbx = " << xbx << "\n"; - // amrex::PrintToFile("loc_multicuts") << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - // << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; - #endif + std::ofstream fout("dat_multicuts.txt", std::ios::app); + fout << "xbx = " << xbx << std::endl; + fout << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + << "(x,y,z) = (" << + problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" + << std::endl; } } @@ -595,17 +589,12 @@ int build_faces (Box const& bx, Array4 const& cell, if (plt_multiple_cuts){ mt_fcy(i,j,k,0) = 10.0; - #ifndef AMREX_USE_GPU - std::ofstream fout("dat_multicuts.txt", std::ios::app); - fout << "ybx = " << ybx << std::endl; - fout << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - << "(x,y,z) = (" << - problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" - << std::endl; - // amrex::PrintToFile("loc_multicuts") << "ybx = " << ybx << "\n"; - // amrex::PrintToFile("loc_multicuts") << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - // << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; - #endif + std::ofstream fout("dat_multicuts.txt", std::ios::app); + fout << "ybx = " << ybx << std::endl; + fout << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + << "(x,y,z) = (" << + problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" + << std::endl; } } @@ -718,17 +707,12 @@ int build_faces (Box const& bx, Array4 const& cell, if (plt_multiple_cuts){ mt_fcz(i,j,k,0) = 10.0; - #ifndef AMREX_USE_GPU - std::ofstream fout("dat_multicuts.txt", std::ios::app); - fout << "zbx = " << zbx << std::endl; - fout << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - << "(x,y,z) = (" << - problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" - << std::endl; - // amrex::PrintToFile("loc_multicuts") << "zbx = " << zbx << "\n"; - // amrex::PrintToFile("loc_multicuts") << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - // << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ") \n"; - #endif + std::ofstream fout("dat_multicuts.txt", std::ios::app); + fout << "zbx = " << zbx << std::endl; + fout << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + << "(x,y,z) = (" << + problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" + << std::endl; } } From 436bb3ff25a89ef2d52ee277ab487d5eb1a3f375 Mon Sep 17 00:00:00 2001 From: Yoo Date: Wed, 28 Aug 2024 15:57:47 -0600 Subject: [PATCH 17/32] Return to PrintToFile since ofstream is not compatible with CUDA --- Src/EB/AMReX_EB2_3D_C.cpp | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 6dbed6bf6cc..baa739cabf3 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -377,10 +377,6 @@ int build_faces (Box const& bx, Array4 const& cell, Gpu::Buffer nmulticuts = {0}; int* hp = nmulticuts.hostData(); int* dp = nmulticuts.data(); - // delete the previous dat_multicuts.txt file - std::ofstream ofs; - ofs.open("dat_multicuts.txt", std::ofstream::out | std::ofstream::trunc); - ofs.close(); #ifdef AMREX_USE_FLOAT constexpr Real small = 1.e-5_rt; @@ -471,12 +467,12 @@ int build_faces (Box const& bx, Array4 const& cell, if (plt_multiple_cuts){ mt_fcx(i,j,k,0) = 10.0; - std::ofstream fout("dat_multicuts.txt", std::ios::app); - fout << "xbx = " << xbx << std::endl; - fout << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + amrex::PrintToFile("loc_multicuts") + << "xbx = " << xbx << "\n" + << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" - << std::endl; + << "\n"; } } @@ -589,12 +585,12 @@ int build_faces (Box const& bx, Array4 const& cell, if (plt_multiple_cuts){ mt_fcy(i,j,k,0) = 10.0; - std::ofstream fout("dat_multicuts.txt", std::ios::app); - fout << "ybx = " << ybx << std::endl; - fout << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + amrex::PrintToFile("loc_multicuts") + << "ybx = " << ybx << "\n" + << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" - << std::endl; + << "\n"; } } @@ -707,12 +703,12 @@ int build_faces (Box const& bx, Array4 const& cell, if (plt_multiple_cuts){ mt_fcz(i,j,k,0) = 10.0; - std::ofstream fout("dat_multicuts.txt", std::ios::app); - fout << "zbx = " << zbx << std::endl; - fout << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + amrex::PrintToFile("loc_multicuts") + << "zbx = " << zbx << "\n" + << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " << "(x,y,z) = (" << problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" - << std::endl; + << "\n"; } } From 0536861ef18bb1acc60744766b709ae417f24f2f Mon Sep 17 00:00:00 2001 From: Yoo Date: Wed, 28 Aug 2024 16:35:26 -0600 Subject: [PATCH 18/32] Forgot to add #ifndef back --- Src/EB/AMReX_EB2_3D_C.cpp | 42 ++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index baa739cabf3..c3e3bf5c69c 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -467,12 +467,14 @@ int build_faces (Box const& bx, Array4 const& cell, if (plt_multiple_cuts){ mt_fcx(i,j,k,0) = 10.0; - amrex::PrintToFile("loc_multicuts") - << "xbx = " << xbx << "\n" - << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - << "(x,y,z) = (" << - problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" - << "\n"; + #ifndef AMREX_USE_OPM + amrex::PrintToFile("loc_multicuts") + << "xbx = " << xbx << "\n" + << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + << "(x,y,z) = (" << + problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" + << "\n"; + #endif } } @@ -585,12 +587,14 @@ int build_faces (Box const& bx, Array4 const& cell, if (plt_multiple_cuts){ mt_fcy(i,j,k,0) = 10.0; - amrex::PrintToFile("loc_multicuts") - << "ybx = " << ybx << "\n" - << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - << "(x,y,z) = (" << - problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" - << "\n"; + #ifndef AMREX_USE_OPM + amrex::PrintToFile("loc_multicuts") + << "ybx = " << ybx << "\n" + << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + << "(x,y,z) = (" << + problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" + << "\n"; + #endif } } @@ -703,12 +707,14 @@ int build_faces (Box const& bx, Array4 const& cell, if (plt_multiple_cuts){ mt_fcz(i,j,k,0) = 10.0; - amrex::PrintToFile("loc_multicuts") - << "zbx = " << zbx << "\n" - << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - << "(x,y,z) = (" << - problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" - << "\n"; + #ifndef AMREX_USE_OPM + amrex::PrintToFile("loc_multicuts") + << "zbx = " << zbx << "\n" + << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " + << "(x,y,z) = (" << + problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" + << "\n"; + #endif } } From 9b635337346c5a19d1129aedb082211157006cdb Mon Sep 17 00:00:00 2001 From: Yoo Date: Wed, 28 Aug 2024 16:40:45 -0600 Subject: [PATCH 19/32] added AMREX_USE_GPU back -> check with Kestrel. is this even compiling ok? --- Src/EB/AMReX_EB2_3D_C.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index c3e3bf5c69c..a8030dd0cab 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -467,7 +467,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (plt_multiple_cuts){ mt_fcx(i,j,k,0) = 10.0; - #ifndef AMREX_USE_OPM + #ifndef AMREX_USE_GPU amrex::PrintToFile("loc_multicuts") << "xbx = " << xbx << "\n" << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " @@ -587,7 +587,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (plt_multiple_cuts){ mt_fcy(i,j,k,0) = 10.0; - #ifndef AMREX_USE_OPM + #ifndef AMREX_USE_GPU amrex::PrintToFile("loc_multicuts") << "ybx = " << ybx << "\n" << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " @@ -707,7 +707,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (plt_multiple_cuts){ mt_fcz(i,j,k,0) = 10.0; - #ifndef AMREX_USE_OPM + #ifndef AMREX_USE_GPU amrex::PrintToFile("loc_multicuts") << "zbx = " << zbx << "\n" << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " From 427d166326dd7a90cce4ceee7883fed61ddd6fd1 Mon Sep 17 00:00:00 2001 From: Yoo Date: Thu, 29 Aug 2024 09:11:13 -0600 Subject: [PATCH 20/32] I just deleted the print txt file part in AMReX_EB2_3D_C.cpp. We may bring it back if we find out a better solution to deal with GPU runs. --- Src/EB/AMReX_EB2_3D_C.cpp | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index a8030dd0cab..70c894decab 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -466,15 +466,6 @@ int build_faces (Box const& bx, Array4 const& cell, Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ mt_fcx(i,j,k,0) = 10.0; - - #ifndef AMREX_USE_GPU - amrex::PrintToFile("loc_multicuts") - << "xbx = " << xbx << "\n" - << "-> fx: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - << "(x,y,z) = (" << - problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" - << "\n"; - #endif } } @@ -586,15 +577,6 @@ int build_faces (Box const& bx, Array4 const& cell, Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ mt_fcy(i,j,k,0) = 10.0; - - #ifndef AMREX_USE_GPU - amrex::PrintToFile("loc_multicuts") - << "ybx = " << ybx << "\n" - << "-> fy: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - << "(x,y,z) = (" << - problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" - << "\n"; - #endif } } @@ -706,15 +688,6 @@ int build_faces (Box const& bx, Array4 const& cell, Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ mt_fcz(i,j,k,0) = 10.0; - - #ifndef AMREX_USE_GPU - amrex::PrintToFile("loc_multicuts") - << "zbx = " << zbx << "\n" - << "-> fz: (i,j,k) = (" << i << ","<< j << "," << k << ") / " - << "(x,y,z) = (" << - problo[0]+(i)*dx[0] << ","<< problo[1]+(j)*dx[1] << "," << problo[2]+(k)*dx[2] << ")" - << "\n"; - #endif } } From df35481f8205582313bda106c74b9eebe8306e22 Mon Sep 17 00:00:00 2001 From: Yoo Date: Thu, 29 Aug 2024 09:45:55 -0600 Subject: [PATCH 21/32] 2D version is added -> how can we chekc? --- Src/EB/AMReX_EB2_2D_C.cpp | 9 ++++++++- Src/EB/AMReX_EB2_C.H | 3 ++- Src/EB/AMReX_EB2_Level.H | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Src/EB/AMReX_EB2_2D_C.cpp b/Src/EB/AMReX_EB2_2D_C.cpp index b2bbde200c5..b32049efcb5 100644 --- a/Src/EB/AMReX_EB2_2D_C.cpp +++ b/Src/EB/AMReX_EB2_2D_C.cpp @@ -204,7 +204,8 @@ int build_faces (Box const& bx, Array4 const& cell, Array4 const& fcx, Array4 const& fcy, GpuArray const& dx, GpuArray const& problo, - bool cover_multiple_cuts, int& nsmallfaces) noexcept + bool cover_multiple_cuts, int& nsmallfaces, + bool plt_multiple_cuts, Array4 const& mt_fcx, Array4 const& mt_fcy) noexcept { #ifdef AMREX_USE_FLOAT constexpr Real small = 1.e-5_rt; @@ -311,6 +312,9 @@ int build_faces (Box const& bx, Array4 const& cell, if (fy(i ,j+1,0) == Type::irregular) { ++ncuts; } if (ncuts > 2) { Gpu::Atomic::Add(dp,1); + if (plt_multiple_cuts){ + mt_fcx(i,j,k,0) = 10.0; + } } } } @@ -334,6 +338,9 @@ int build_faces (Box const& bx, Array4 const& cell, { levset(i,j,k) = Real(0.0); Gpu::Atomic::Add(dp+1,1); + if (plt_multiple_cuts){ + mt_fcy(i,j,k,0) = 10.0; + } } } }); diff --git a/Src/EB/AMReX_EB2_C.H b/Src/EB/AMReX_EB2_C.H index 3f31d883574..e0c931aeef6 100644 --- a/Src/EB/AMReX_EB2_C.H +++ b/Src/EB/AMReX_EB2_C.H @@ -25,7 +25,8 @@ int build_faces (Box const& bx, Array4 const& cell, Array4 const& fcx, Array4 const& fcy, GpuArray const& dx, GpuArray const& problo, - bool cover_multiple_cuts, int& nsmallfaces) noexcept; + bool cover_multiple_cuts, int& nsmallfaces, + bool plt_multiple_cuts, Array4 const& mt_fcx, Array4 const& mt_fcy) noexcept; void build_cells (Box const& bx, Array4 const& cell, Array4 const& fx, Array4 const& fy, diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index e78515caaaa..b96662f6987 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -485,7 +485,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, } nmc = build_faces(vbx, cfg, ftx, fty, lst, xip, yip, apx, apy, fcx, fcy, - dx, problo, cover_multiple_cuts, nsm); + dx, problo, cover_multiple_cuts, nsm, plt_multiple_cuts, multicut_arr_x, multicut_arr_y); build_cells(vbx, cfg, ftx, fty, apx, apy, dx, vfr, ctr, bar, bct, bnm, lst, small_volfrac, geom, extend_domain_face, From bb08807cb65c6808a04fbbf54a0e1541bd7ebf27 Mon Sep 17 00:00:00 2001 From: Yoo Date: Thu, 29 Aug 2024 10:53:17 -0600 Subject: [PATCH 22/32] Whitespace edit --- Src/EB/AMReX_EB2_Level.H | 2 -- 1 file changed, 2 deletions(-) diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index b96662f6987..9346a6448a2 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -106,7 +106,6 @@ protected: private: template friend class GShopLevel; - // Need this function to work around a gcc bug. void setRegularLevel () { m_allregular = true; @@ -497,7 +496,6 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, } ParallelAllReduce::Sum({nsmallcells,nmulticuts}, ParallelContext::CommunicatorSub()); - if (plt_multiple_cuts && nmulticuts > 0) { amrex::Print() << "Total number of multicuts = " << nmulticuts << "\n"; From 3c0b9ce37dde4d611ab02932bb4b15c40eb05807 Mon Sep 17 00:00:00 2001 From: Yoo Date: Thu, 29 Aug 2024 13:56:07 -0600 Subject: [PATCH 23/32] Fixed 2D case error - --- Src/EB/AMReX_EB2_Level.H | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 9346a6448a2..7e7ceaf71a4 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -367,10 +367,6 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Array4 const& bct = m_bndrycent.array(mfi); Array4 const& bnm = m_bndrynorm.array(mfi); - Array4 const& multicut_arr_x = multicut_fcx.array(mfi); - Array4 const& multicut_arr_y = multicut_fcy.array(mfi); - Array4 const& multicut_arr_z = multicut_fcz.array(mfi); - AMREX_D_TERM(Array4 const& apx = m_areafrac[0].array(mfi);, Array4 const& apy = m_areafrac[1].array(mfi);, Array4 const& apz = m_areafrac[2].array(mfi);); @@ -430,6 +426,10 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Array4 const& ym2 = M2[1].array(); Array4 const& zm2 = M2[2].array(); + Array4 const& multicut_arr_x = multicut_fcx.array(mfi); + Array4 const& multicut_arr_y = multicut_fcy.array(mfi); + Array4 const& multicut_arr_z = multicut_fcz.array(mfi); + nmc = build_faces(vbx, cfg, ftx, fty, ftz, xdg, ydg, zdg, lst, xip, yip, zip, apx, apy, apz, fcx, fcy, fcz, xm2, ym2, zm2, dx, problo, cover_multiple_cuts, plt_multiple_cuts, multicut_arr_x, multicut_arr_y, multicut_arr_z); @@ -483,6 +483,9 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, clst, geom); } + Array4 const& multicut_arr_x = multicut_fcx.array(mfi); + Array4 const& multicut_arr_y = multicut_fcy.array(mfi); + nmc = build_faces(vbx, cfg, ftx, fty, lst, xip, yip, apx, apy, fcx, fcy, dx, problo, cover_multiple_cuts, nsm, plt_multiple_cuts, multicut_arr_x, multicut_arr_y); From 7db7054266a8a2c0d4b03b76beb1ef5c38303f81 Mon Sep 17 00:00:00 2001 From: Eunji Yoo Date: Fri, 30 Aug 2024 10:36:12 -0600 Subject: [PATCH 24/32] Fixed whitespace things --- Src/EB/AMReX_EB2_3D_C.cpp | 2 +- Src/EB/AMReX_EB2_C.H | 2 +- Src/EB/AMReX_EB2_Level.H | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 70c894decab..9073d418429 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include namespace amrex::EB2 { diff --git a/Src/EB/AMReX_EB2_C.H b/Src/EB/AMReX_EB2_C.H index e0c931aeef6..140b24ffc6a 100644 --- a/Src/EB/AMReX_EB2_C.H +++ b/Src/EB/AMReX_EB2_C.H @@ -71,7 +71,7 @@ void build_cells (Box const& bx, Array4 const& cell, Array4 const& barea, Array4 const& bcent, Array4 const& bnorm, Array4 const& ctmp, Array4 const& levset, Real small_volfrac, Geometry const& geom, - bool extend_domain_face, bool cover_multiple_cuts, bool plt_multiple_cuts, + bool extend_domain_face, bool cover_multiple_cuts, bool plt_multiple_cuts, int& nsmallcells, int& nmulticuts) noexcept; void set_connection_flags(Box const& bx, Box const& bxg1, diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 7e7ceaf71a4..600132b0686 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -172,14 +172,14 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Real small_volfrac = 1.e-14; #endif bool cover_multiple_cuts = false; - bool plt_multiple_cuts = false; + bool plt_multiple_cuts = false; int maxiter = 32; { ParmParse pp("eb2"); pp.queryAdd("small_volfrac", small_volfrac); pp.queryAdd("cover_multiple_cuts", cover_multiple_cuts); pp.queryAdd("maxiter", maxiter); - pp.queryAdd("plt_multiple_cuts", plt_multiple_cuts); + pp.queryAdd("plt_multiple_cuts", plt_multiple_cuts); } maxiter = std::min(100000, maxiter); @@ -298,9 +298,9 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, m_bndrycent.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); m_bndrynorm.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); - multicut_fcx.define(m_grids, m_dmap, 1, ng, mf_info); - multicut_fcy.define(m_grids, m_dmap, 1, ng, mf_info); - multicut_fcz.define(m_grids, m_dmap, 1, ng, mf_info); + multicut_fcx.define(m_grids, m_dmap, 1, ng, mf_info); + multicut_fcy.define(m_grids, m_dmap, 1, ng, mf_info); + multicut_fcz.define(m_grids, m_dmap, 1, ng, mf_info); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_areafrac[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), @@ -501,14 +501,14 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, ParallelAllReduce::Sum({nsmallcells,nmulticuts}, ParallelContext::CommunicatorSub()); if (plt_multiple_cuts && nmulticuts > 0) { - amrex::Print() << "Total number of multicuts = " << nmulticuts << "\n"; + amrex::Print() << "Total number of multicuts = " << nmulticuts << "\n"; amrex::Print() << "plotting multicut locations..." << "\n"; WriteSingleLevelPlotfile("plt.x", multicut_fcx, {"multicut_x"}, geom, 0.0, 0); WriteSingleLevelPlotfile("plt.y", multicut_fcy, {"multicut_y"}, geom, 0.0, 0); WriteSingleLevelPlotfile("plt.z", multicut_fcz, {"multicut_z"}, geom, 0.0, 0); - - amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + + amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); } if (nsmallcells == 0 && nmulticuts == 0) { From 3aeff0e970e0fc4ec77575530546fc92f2db5270 Mon Sep 17 00:00:00 2001 From: Yoo Date: Fri, 30 Aug 2024 11:40:44 -0600 Subject: [PATCH 25/32] RE-generate the MF for face-centered one. called multicut_face. 2D is also implemented --- Src/EB/AMReX_EB2_3D_C.cpp | 8 ++++---- Src/EB/AMReX_EB2_Level.H | 34 +++++++++++++++------------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 70c894decab..9dd5c7d3596 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -372,7 +372,7 @@ int build_faces (Box const& bx, Array4 const& cell, GpuArray const& dx, GpuArray const& problo, bool cover_multiple_cuts, - bool plt_multiple_cuts, Array4 const& mt_fcx, Array4 const& mt_fcy, Array4 const& mt_fcz) noexcept + bool plt_multiple_cuts, Array4 const& mcx, Array4 const& mcy, Array4 const& mcz) noexcept { Gpu::Buffer nmulticuts = {0}; int* hp = nmulticuts.hostData(); @@ -465,7 +465,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - mt_fcx(i,j,k,0) = 10.0; + mcx(i,j,k) = 10.0; } } @@ -576,7 +576,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - mt_fcy(i,j,k,0) = 10.0; + mcy(i,j,k) = 10.0; } } @@ -687,7 +687,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - mt_fcz(i,j,k,0) = 10.0; + mcz(i,j,k) = 10.0; } } diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 7e7ceaf71a4..865480d25ab 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -49,6 +49,8 @@ public: void fillBndryNorm ( MultiFab& bndrynorm, const Geometry& geom) const; void fillAreaFrac (Array const& areafrac, const Geometry& geom) const; void fillAreaFrac (Array< MultiFab*,AMREX_SPACEDIM> const& areafrac, const Geometry& geom) const; + void fillFaceMulticut (Array const& multicut_face, const Geometry& geom) const; + void fillFaceMulticut (Array< MultiFab*,AMREX_SPACEDIM> const& multicut_face, const Geometry& geom) const; void fillFaceCent (Array const& facecent, const Geometry& geom) const; void fillFaceCent (Array< MultiFab*,AMREX_SPACEDIM> const& facecent, const Geometry& geom) const; void fillEdgeCent (Array const& edgecent, const Geometry& geom) const; @@ -92,6 +94,7 @@ protected: MultiFab m_bndrycent; MultiFab m_bndrynorm; Array m_areafrac; + Array multicut_face; Array m_facecent; Array m_edgecent; iMultiFab m_cutcellmask; @@ -100,10 +103,6 @@ protected: bool m_has_eb_info = true; IndexSpace const* m_parent; - MultiFab multicut_fcx; - MultiFab multicut_fcy; - MultiFab multicut_fcz; - private: template friend class GShopLevel; // Need this function to work around a gcc bug. @@ -298,13 +297,11 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, m_bndrycent.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); m_bndrynorm.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); - multicut_fcx.define(m_grids, m_dmap, 1, ng, mf_info); - multicut_fcy.define(m_grids, m_dmap, 1, ng, mf_info); - multicut_fcz.define(m_grids, m_dmap, 1, ng, mf_info); - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_areafrac[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, 1, ng, mf_info); + multicut_face[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), + m_dmap, 1, ng, mf_info); m_facecent[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, AMREX_SPACEDIM-1, ng, mf_info); IntVect edge_type{1}; edge_type[idim] = 0; @@ -370,6 +367,9 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, AMREX_D_TERM(Array4 const& apx = m_areafrac[0].array(mfi);, Array4 const& apy = m_areafrac[1].array(mfi);, Array4 const& apz = m_areafrac[2].array(mfi);); + AMREX_D_TERM(Array4 const& mcx = multicut_face[0].array(mfi);, + Array4 const& mcy = multicut_face[1].array(mfi);, + Array4 const& mcz = multicut_face[2].array(mfi);); AMREX_D_TERM(Array4 const& fcx = m_facecent[0].array(mfi);, Array4 const& fcy = m_facecent[1].array(mfi);, Array4 const& fcz = m_facecent[2].array(mfi);); @@ -426,13 +426,9 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Array4 const& ym2 = M2[1].array(); Array4 const& zm2 = M2[2].array(); - Array4 const& multicut_arr_x = multicut_fcx.array(mfi); - Array4 const& multicut_arr_y = multicut_fcy.array(mfi); - Array4 const& multicut_arr_z = multicut_fcz.array(mfi); - nmc = build_faces(vbx, cfg, ftx, fty, ftz, xdg, ydg, zdg, lst, xip, yip, zip, apx, apy, apz, fcx, fcy, fcz, - xm2, ym2, zm2, dx, problo, cover_multiple_cuts, plt_multiple_cuts, multicut_arr_x, multicut_arr_y, multicut_arr_z); + xm2, ym2, zm2, dx, problo, cover_multiple_cuts, plt_multiple_cuts, mcx, mcy, mcz); cellflagtmp.resize(m_cellflag[mfi].box()); Elixir cellflagtmp_eli = cellflagtmp.elixir(); @@ -483,11 +479,11 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, clst, geom); } - Array4 const& multicut_arr_x = multicut_fcx.array(mfi); - Array4 const& multicut_arr_y = multicut_fcy.array(mfi); + AMREX_D_TERM(Array4 const& mcx = multicut_face[0].array(mfi);, + Array4 const& mcy = multicut_face[1].array(mfi));, nmc = build_faces(vbx, cfg, ftx, fty, lst, xip, yip, apx, apy, fcx, fcy, - dx, problo, cover_multiple_cuts, nsm, plt_multiple_cuts, multicut_arr_x, multicut_arr_y); + dx, problo, cover_multiple_cuts, nsm, plt_multiple_cuts, mcx, mcy); build_cells(vbx, cfg, ftx, fty, apx, apy, dx, vfr, ctr, bar, bct, bnm, lst, small_volfrac, geom, extend_domain_face, @@ -504,9 +500,9 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, amrex::Print() << "Total number of multicuts = " << nmulticuts << "\n"; amrex::Print() << "plotting multicut locations..." << "\n"; - WriteSingleLevelPlotfile("plt.x", multicut_fcx, {"multicut_x"}, geom, 0.0, 0); - WriteSingleLevelPlotfile("plt.y", multicut_fcy, {"multicut_y"}, geom, 0.0, 0); - WriteSingleLevelPlotfile("plt.z", multicut_fcz, {"multicut_z"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.x", multicut_face[0], {"multicut_fcx"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.y", multicut_face[1], {"multicut_fcy"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.z", multicut_face[2], {"multicut_fcz"}, geom, 0.0, 0); amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); } From 002386948a8854a5b26d92b1ba8af16745e61def Mon Sep 17 00:00:00 2001 From: Yoo Date: Fri, 30 Aug 2024 15:48:24 -0600 Subject: [PATCH 26/32] 1) fixed 2D bug 2) added if-conditions for plotting only when we have multicuts for each direction 3) face-centered multicut --- Src/EB/AMReX_EB2_2D_C.cpp | 15 +++++++++------ Src/EB/AMReX_EB2_3D_C.cpp | 4 ++-- Src/EB/AMReX_EB2_C.H | 3 ++- Src/EB/AMReX_EB2_Level.H | 31 ++++++++++++++++++++++--------- 4 files changed, 35 insertions(+), 18 deletions(-) diff --git a/Src/EB/AMReX_EB2_2D_C.cpp b/Src/EB/AMReX_EB2_2D_C.cpp index b32049efcb5..1d35061ed42 100644 --- a/Src/EB/AMReX_EB2_2D_C.cpp +++ b/Src/EB/AMReX_EB2_2D_C.cpp @@ -205,7 +205,7 @@ int build_faces (Box const& bx, Array4 const& cell, GpuArray const& dx, GpuArray const& problo, bool cover_multiple_cuts, int& nsmallfaces, - bool plt_multiple_cuts, Array4 const& mt_fcx, Array4 const& mt_fcy) noexcept + bool plt_multiple_cuts, Array4 const& mt_fcx) noexcept { #ifdef AMREX_USE_FLOAT constexpr Real small = 1.e-5_rt; @@ -313,7 +313,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - mt_fcx(i,j,k,0) = 10.0; + mt_fcx(i,j,k) = 10.0; } } } @@ -338,9 +338,6 @@ int build_faces (Box const& bx, Array4 const& cell, { levset(i,j,k) = Real(0.0); Gpu::Atomic::Add(dp+1,1); - if (plt_multiple_cuts){ - mt_fcy(i,j,k,0) = 10.0; - } } } }); @@ -350,7 +347,12 @@ int build_faces (Box const& bx, Array4 const& cell, nsmallfaces += *(hp+1); if (*hp > 0 && !cover_multiple_cuts) { - amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + if (plt_multiple_cuts){ + amrex::Print() << "Passing EB2::build_cells" << "\n"; + } + else{ + amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + } } return *hp; @@ -364,6 +366,7 @@ void build_cells (Box const& bx, Array4 const& cell, Array4 const& barea, Array4 const& bcent, Array4 const& bnorm, Array4 const& levset, Real small_volfrac, Geometry const& geom, bool extend_domain_face, + bool plt_multiple_cuts, int& nsmallcells, int const nmulticuts) noexcept { Gpu::Buffer smc = {0}; diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 87e1d5414ec..f158e89afcc 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -576,7 +576,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - mcy(i,j,k) = 10.0; + mcy(i,j,k) = 20.0; } } @@ -687,7 +687,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - mcz(i,j,k) = 10.0; + mcz(i,j,k) = 30.0; } } diff --git a/Src/EB/AMReX_EB2_C.H b/Src/EB/AMReX_EB2_C.H index 140b24ffc6a..492e748131c 100644 --- a/Src/EB/AMReX_EB2_C.H +++ b/Src/EB/AMReX_EB2_C.H @@ -26,7 +26,7 @@ int build_faces (Box const& bx, Array4 const& cell, GpuArray const& dx, GpuArray const& problo, bool cover_multiple_cuts, int& nsmallfaces, - bool plt_multiple_cuts, Array4 const& mt_fcx, Array4 const& mt_fcy) noexcept; + bool plt_multiple_cuts, Array4 const& mt_fcx) noexcept; void build_cells (Box const& bx, Array4 const& cell, Array4 const& fx, Array4 const& fy, @@ -36,6 +36,7 @@ void build_cells (Box const& bx, Array4 const& cell, Array4 const& barea, Array4 const& bcent, Array4 const& bnorm, Array4 const& levset, Real small_volfrac, Geometry const& geom, bool extend_domain_face, + bool plt_multiple_cuts, int& nsmallcells, int nmulticuts) noexcept; void set_connection_flags(Box const& bxg1, Array4 const& cell, diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index bb77eeb7c22..271f0cd7462 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -479,14 +479,11 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, clst, geom); } - AMREX_D_TERM(Array4 const& mcx = multicut_face[0].array(mfi);, - Array4 const& mcy = multicut_face[1].array(mfi));, - nmc = build_faces(vbx, cfg, ftx, fty, lst, xip, yip, apx, apy, fcx, fcy, - dx, problo, cover_multiple_cuts, nsm, plt_multiple_cuts, mcx, mcy); + dx, problo, cover_multiple_cuts, nsm, plt_multiple_cuts, mcx); build_cells(vbx, cfg, ftx, fty, apx, apy, dx, vfr, ctr, - bar, bct, bnm, lst, small_volfrac, geom, extend_domain_face, + bar, bct, bnm, lst, small_volfrac, geom, extend_domain_face, plt_multiple_cuts, nsm, nmc); #endif nsmallcells += nsm; @@ -500,10 +497,26 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, amrex::Print() << "Total number of multicuts = " << nmulticuts << "\n"; amrex::Print() << "plotting multicut locations..." << "\n"; - WriteSingleLevelPlotfile("plt.x", multicut_face[0], {"multicut_fcx"}, geom, 0.0, 0); - WriteSingleLevelPlotfile("plt.y", multicut_face[1], {"multicut_fcy"}, geom, 0.0, 0); - WriteSingleLevelPlotfile("plt.z", multicut_face[2], {"multicut_fcz"}, geom, 0.0, 0); - + #if (AMREX_SPACEDIM == 3) + if (!multicut_face[0].empty()) + { + WriteSingleLevelPlotfile("plt.x", multicut_face[0], {"multicut_fcx"}, geom, 0.0, 0); + } + if (!multicut_face[1].empty()) + { + WriteSingleLevelPlotfile("plt.y", multicut_face[1], {"multicut_fcy"}, geom, 0.0, 0); + } + if (!multicut_face[2].empty()) + { + WriteSingleLevelPlotfile("plt.z", multicut_face[2], {"multicut_fcz"}, geom, 0.0, 0); + } + #elif (AMREX_SPACEDIM == 2) + if (!multicut_face[0].empty()) + { + WriteSingleLevelPlotfile("plt.x", multicut_face[0], {"multicut_fcx"}, geom, 0.0, 0); + } + #endif + amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); } From 0f199153bcdef41f44107b6ae907a801f1594881 Mon Sep 17 00:00:00 2001 From: Eunji Yoo Date: Fri, 30 Aug 2024 16:08:38 -0600 Subject: [PATCH 27/32] Whitespace sytling --- Src/EB/AMReX_EB2_Level.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 271f0cd7462..22278d5b248 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -517,7 +517,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, } #endif - amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); } if (nsmallcells == 0 && nmulticuts == 0) { From fb47a1ab0c9cc4624c6dcc87df473fcfb132a698 Mon Sep 17 00:00:00 2001 From: Yoo Date: Thu, 5 Sep 2024 19:43:08 -0700 Subject: [PATCH 28/32] changed the location to call mcx,y,z arrays -> 2D is checked as well --- Src/EB/AMReX_EB2_Level.H | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 22278d5b248..90f0501ac3b 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -367,9 +367,6 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, AMREX_D_TERM(Array4 const& apx = m_areafrac[0].array(mfi);, Array4 const& apy = m_areafrac[1].array(mfi);, Array4 const& apz = m_areafrac[2].array(mfi);); - AMREX_D_TERM(Array4 const& mcx = multicut_face[0].array(mfi);, - Array4 const& mcy = multicut_face[1].array(mfi);, - Array4 const& mcz = multicut_face[2].array(mfi);); AMREX_D_TERM(Array4 const& fcx = m_facecent[0].array(mfi);, Array4 const& fcy = m_facecent[1].array(mfi);, Array4 const& fcz = m_facecent[2].array(mfi);); @@ -426,6 +423,10 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Array4 const& ym2 = M2[1].array(); Array4 const& zm2 = M2[2].array(); + AMREX_D_TERM(Array4 const& mcx = multicut_face[0].array(mfi);, + Array4 const& mcy = multicut_face[1].array(mfi);, + Array4 const& mcz = multicut_face[2].array(mfi);); + nmc = build_faces(vbx, cfg, ftx, fty, ftz, xdg, ydg, zdg, lst, xip, yip, zip, apx, apy, apz, fcx, fcy, fcz, xm2, ym2, zm2, dx, problo, cover_multiple_cuts, plt_multiple_cuts, mcx, mcy, mcz); @@ -479,6 +480,8 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, clst, geom); } + Array4 const& mcx = multicut_face[0].array(mfi); + nmc = build_faces(vbx, cfg, ftx, fty, lst, xip, yip, apx, apy, fcx, fcy, dx, problo, cover_multiple_cuts, nsm, plt_multiple_cuts, mcx); From a0dc9c6a1679569864d9290f0e7e13e533102c40 Mon Sep 17 00:00:00 2001 From: Yoo Date: Fri, 6 Sep 2024 08:52:35 -0700 Subject: [PATCH 29/32] fixed notations in .H files --- Src/EB/AMReX_EB2_2D_C.cpp | 5 ++--- Src/EB/AMReX_EB2_C.H | 5 ++--- Src/EB/AMReX_EB2_Level.H | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Src/EB/AMReX_EB2_2D_C.cpp b/Src/EB/AMReX_EB2_2D_C.cpp index 1d35061ed42..3a603a15980 100644 --- a/Src/EB/AMReX_EB2_2D_C.cpp +++ b/Src/EB/AMReX_EB2_2D_C.cpp @@ -205,7 +205,7 @@ int build_faces (Box const& bx, Array4 const& cell, GpuArray const& dx, GpuArray const& problo, bool cover_multiple_cuts, int& nsmallfaces, - bool plt_multiple_cuts, Array4 const& mt_fcx) noexcept + bool plt_multiple_cuts, Array4 const& mcx) noexcept { #ifdef AMREX_USE_FLOAT constexpr Real small = 1.e-5_rt; @@ -313,7 +313,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - mt_fcx(i,j,k) = 10.0; + mcx(i,j,k) = 10.0; } } } @@ -366,7 +366,6 @@ void build_cells (Box const& bx, Array4 const& cell, Array4 const& barea, Array4 const& bcent, Array4 const& bnorm, Array4 const& levset, Real small_volfrac, Geometry const& geom, bool extend_domain_face, - bool plt_multiple_cuts, int& nsmallcells, int const nmulticuts) noexcept { Gpu::Buffer smc = {0}; diff --git a/Src/EB/AMReX_EB2_C.H b/Src/EB/AMReX_EB2_C.H index 492e748131c..37d5d49b08c 100644 --- a/Src/EB/AMReX_EB2_C.H +++ b/Src/EB/AMReX_EB2_C.H @@ -26,7 +26,7 @@ int build_faces (Box const& bx, Array4 const& cell, GpuArray const& dx, GpuArray const& problo, bool cover_multiple_cuts, int& nsmallfaces, - bool plt_multiple_cuts, Array4 const& mt_fcx) noexcept; + bool plt_multiple_cuts, Array4 const& mcx) noexcept; void build_cells (Box const& bx, Array4 const& cell, Array4 const& fx, Array4 const& fy, @@ -36,7 +36,6 @@ void build_cells (Box const& bx, Array4 const& cell, Array4 const& barea, Array4 const& bcent, Array4 const& bnorm, Array4 const& levset, Real small_volfrac, Geometry const& geom, bool extend_domain_face, - bool plt_multiple_cuts, int& nsmallcells, int nmulticuts) noexcept; void set_connection_flags(Box const& bxg1, Array4 const& cell, @@ -58,7 +57,7 @@ int build_faces (Box const& bx, Array4 const& cell, GpuArray const& dx, GpuArray const& problo, bool cover_multiple_cuts, - bool plt_multiple_cuts, Array4 const& mt_fcx, Array4 const& mt_fcy, Array4 const& mt_fcz) noexcept; + bool plt_multiple_cuts, Array4 const& mcx, Array4 const& mcy, Array4 const& mcz) noexcept; void build_cells (Box const& bx, Array4 const& cell, Array4 const& fx, Array4 const& fy, diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 90f0501ac3b..3ad4d2183c3 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -486,7 +486,7 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, dx, problo, cover_multiple_cuts, nsm, plt_multiple_cuts, mcx); build_cells(vbx, cfg, ftx, fty, apx, apy, dx, vfr, ctr, - bar, bct, bnm, lst, small_volfrac, geom, extend_domain_face, plt_multiple_cuts, + bar, bct, bnm, lst, small_volfrac, geom, extend_domain_face, nsm, nmc); #endif nsmallcells += nsm; From 8b4c582bc950218e93b31df7b41b5f81e740f8f5 Mon Sep 17 00:00:00 2001 From: Yoo Date: Mon, 9 Sep 2024 10:51:20 -0700 Subject: [PATCH 30/32] Edited based on Bruce's comments on Sep 6. --- Src/EB/AMReX_EB2_2D_C.cpp | 7 ++----- Src/EB/AMReX_EB2_3D_C.cpp | 19 ++++++------------- Src/EB/AMReX_EB2_Level.H | 11 +++++------ 3 files changed, 13 insertions(+), 24 deletions(-) diff --git a/Src/EB/AMReX_EB2_2D_C.cpp b/Src/EB/AMReX_EB2_2D_C.cpp index 3a603a15980..49bb6047610 100644 --- a/Src/EB/AMReX_EB2_2D_C.cpp +++ b/Src/EB/AMReX_EB2_2D_C.cpp @@ -313,7 +313,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - mcx(i,j,k) = 10.0; + mcx(i,j,k) = ncuts; } } } @@ -347,10 +347,7 @@ int build_faces (Box const& bx, Array4 const& cell, nsmallfaces += *(hp+1); if (*hp > 0 && !cover_multiple_cuts) { - if (plt_multiple_cuts){ - amrex::Print() << "Passing EB2::build_cells" << "\n"; - } - else{ + if (!plt_multiple_cuts){ amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); } } diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index f158e89afcc..6b4ad5e71e2 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -388,7 +388,7 @@ int build_faces (Box const& bx, Array4 const& cell, const Real dzinv = 1.0_rt/dx[2]; const Box& xbx = amrex::grow(amrex::surroundingNodes(bx,0),1); - AMREX_HOST_DEVICE_FOR_3D ( xbx, i, j, k, // this is face index + AMREX_HOST_DEVICE_FOR_3D ( xbx, i, j, k, { if (fx(i,j,k) == Type::regular) { apx(i,j,k) = 1.0_rt; @@ -465,7 +465,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - mcx(i,j,k) = 10.0; + mcx(i,j,k) = ncuts; } } @@ -576,7 +576,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - mcy(i,j,k) = 20.0; + mcy(i,j,k) = ncuts; } } @@ -687,7 +687,7 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); if (plt_multiple_cuts){ - mcz(i,j,k) = 30.0; + mcz(i,j,k) = ncuts; } } @@ -779,11 +779,7 @@ int build_faces (Box const& bx, Array4 const& cell, } }); } else { - // Passing now to plot the location of multicuts before aborting! - if (plt_multiple_cuts){ - amrex::Print() << "Creating outputs for multicut locations..." << "\n"; - } - else{ + if (!plt_multiple_cuts){ amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported!"); } } @@ -949,10 +945,7 @@ void build_cells (Box const& bx, Array4 const& cell, if (nsmallcells > 0 || nmulticuts > 0) { if (!cover_multiple_cuts && nmulticuts > 0) { - if (plt_multiple_cuts){ - amrex::Print() << "Passing EB2::build_cells" << "\n"; - } - else{ + if (!plt_multiple_cuts){ amrex::Abort("amrex::EB2::build_cells: multi-cuts not supported"); } } diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 3ad4d2183c3..5eb269b68a3 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -503,24 +503,23 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, #if (AMREX_SPACEDIM == 3) if (!multicut_face[0].empty()) { - WriteSingleLevelPlotfile("plt.x", multicut_face[0], {"multicut_fcx"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.multicut.x", multicut_face[0], {"multicut_fcx"}, geom, 0.0, 0); } if (!multicut_face[1].empty()) { - WriteSingleLevelPlotfile("plt.y", multicut_face[1], {"multicut_fcy"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.multicut.y", multicut_face[1], {"multicut_fcy"}, geom, 0.0, 0); } if (!multicut_face[2].empty()) { - WriteSingleLevelPlotfile("plt.z", multicut_face[2], {"multicut_fcz"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.multicut.z", multicut_face[2], {"multicut_fcz"}, geom, 0.0, 0); } #elif (AMREX_SPACEDIM == 2) if (!multicut_face[0].empty()) { - WriteSingleLevelPlotfile("plt.x", multicut_face[0], {"multicut_fcx"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.multicut", multicut_face[0], {"multicut_fcx"}, geom, 0.0, 0); } #endif - - amrex::Abort("amrex::EB2::build_faces: more than 2 cuts not supported"); + amrex::Abort("amrex::EB2:: more than 2 cuts not supported"); } if (nsmallcells == 0 && nmulticuts == 0) { From 766e8a948ad706d7957b11d944b2b6e7e192d07c Mon Sep 17 00:00:00 2001 From: Yoo Date: Mon, 9 Sep 2024 15:49:37 -0700 Subject: [PATCH 31/32] changed the way we fill the plot file data: Value of 2 is regular, 0 (or very tiny) is covered, and some cell values > 2 means multicuts --- Src/EB/AMReX_EB2_2D_C.cpp | 6 +++--- Src/EB/AMReX_EB2_3D_C.cpp | 21 ++++++++++++--------- Src/EB/AMReX_EB2_Level.H | 11 ++++++----- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/Src/EB/AMReX_EB2_2D_C.cpp b/Src/EB/AMReX_EB2_2D_C.cpp index e39a38d7ab3..649e1fe563a 100644 --- a/Src/EB/AMReX_EB2_2D_C.cpp +++ b/Src/EB/AMReX_EB2_2D_C.cpp @@ -311,10 +311,10 @@ int build_faces (Box const& bx, Array4 const& cell, if (fy(i ,j+1,0) == Type::irregular) { ++ncuts; } if (ncuts > 2) { Gpu::Atomic::Add(dp,1); - if (plt_multiple_cuts){ - mcx(i,j,k) = ncuts; - } } + if (plt_multiple_cuts){ + mcx(i,j,k) = ncuts; + } } } }); diff --git a/Src/EB/AMReX_EB2_3D_C.cpp b/Src/EB/AMReX_EB2_3D_C.cpp index 75da10766ee..94386b10323 100644 --- a/Src/EB/AMReX_EB2_3D_C.cpp +++ b/Src/EB/AMReX_EB2_3D_C.cpp @@ -466,9 +466,6 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); - if (plt_multiple_cuts){ - mcx(i,j,k) = ncuts; - } } if ((ncuts > 2) || (lym <= small && lyp <= small && lzm <= small && lzp <= small)) { @@ -498,6 +495,10 @@ int build_faces (Box const& bx, Array4 const& cell, } else if (apx(i,j,k) == 1.0_rt) { fx(i,j,k) = Type::regular; } + + if (plt_multiple_cuts){ + mcx(i,j,k) = ncuts; + } } }); @@ -577,9 +578,6 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); - if (plt_multiple_cuts){ - mcy(i,j,k) = ncuts; - } } if ((ncuts > 2) || (lxm <= small && lxp <= small && lzm <= small && lzp <= small)) { @@ -609,6 +607,10 @@ int build_faces (Box const& bx, Array4 const& cell, } else if (apy(i,j,k) == 1.0_rt) { fy(i,j,k) = Type::regular; } + + if (plt_multiple_cuts){ + mcy(i,j,k) = ncuts; + } } }); @@ -688,9 +690,6 @@ int build_faces (Box const& bx, Array4 const& cell, if (ncuts > 2) { Gpu::Atomic::Add(dp,1); - if (plt_multiple_cuts){ - mcz(i,j,k) = ncuts; - } } if ((ncuts > 2) || (lxm <= small && lxp <= small && lym <= small && lyp <= small)) { @@ -720,6 +719,10 @@ int build_faces (Box const& bx, Array4 const& cell, } else if (apz(i,j,k) == 1.0_rt) { fz(i,j,k) = Type::regular; } + + if (plt_multiple_cuts){ + mcz(i,j,k) = ncuts; + } } }); diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 5eb269b68a3..65b1644ce63 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -498,23 +498,24 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, if (plt_multiple_cuts && nmulticuts > 0) { amrex::Print() << "Total number of multicuts = " << nmulticuts << "\n"; - amrex::Print() << "plotting multicut locations..." << "\n"; + amrex::Print() << "Plotting multicut locations..." << "\n"; + amrex::Print() << "Max is = " << multicut_face[0].max(0) << "\n"; #if (AMREX_SPACEDIM == 3) - if (!multicut_face[0].empty()) + if (multicut_face[0].max(0) > 2) { WriteSingleLevelPlotfile("plt.multicut.x", multicut_face[0], {"multicut_fcx"}, geom, 0.0, 0); } - if (!multicut_face[1].empty()) + if (multicut_face[1].max(0) > 2) { WriteSingleLevelPlotfile("plt.multicut.y", multicut_face[1], {"multicut_fcy"}, geom, 0.0, 0); } - if (!multicut_face[2].empty()) + if (multicut_face[2].max(0) > 2) { WriteSingleLevelPlotfile("plt.multicut.z", multicut_face[2], {"multicut_fcz"}, geom, 0.0, 0); } #elif (AMREX_SPACEDIM == 2) - if (!multicut_face[0].empty()) + if (multicut_face[0].max(0) > 2) { WriteSingleLevelPlotfile("plt.multicut", multicut_face[0], {"multicut_fcx"}, geom, 0.0, 0); } From 8eb29d88bcbf4601b35301c8261552a3fec31887 Mon Sep 17 00:00:00 2001 From: Yoo Date: Thu, 19 Sep 2024 14:48:57 -0700 Subject: [PATCH 32/32] make chage to cell-centered MF --- Src/EB/AMReX_EB2_Level.H | 52 +++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/Src/EB/AMReX_EB2_Level.H b/Src/EB/AMReX_EB2_Level.H index 65b1644ce63..695f9bb02dc 100644 --- a/Src/EB/AMReX_EB2_Level.H +++ b/Src/EB/AMReX_EB2_Level.H @@ -14,7 +14,6 @@ #include #include #include - #include #ifdef AMREX_USE_OMP @@ -49,8 +48,6 @@ public: void fillBndryNorm ( MultiFab& bndrynorm, const Geometry& geom) const; void fillAreaFrac (Array const& areafrac, const Geometry& geom) const; void fillAreaFrac (Array< MultiFab*,AMREX_SPACEDIM> const& areafrac, const Geometry& geom) const; - void fillFaceMulticut (Array const& multicut_face, const Geometry& geom) const; - void fillFaceMulticut (Array< MultiFab*,AMREX_SPACEDIM> const& multicut_face, const Geometry& geom) const; void fillFaceCent (Array const& facecent, const Geometry& geom) const; void fillFaceCent (Array< MultiFab*,AMREX_SPACEDIM> const& facecent, const Geometry& geom) const; void fillEdgeCent (Array const& edgecent, const Geometry& geom) const; @@ -94,9 +91,11 @@ protected: MultiFab m_bndrycent; MultiFab m_bndrynorm; Array m_areafrac; - Array multicut_face; Array m_facecent; Array m_edgecent; + MultiFab m_multicut_fcx; + MultiFab m_multicut_fcy; + MultiFab m_multicut_fcz; iMultiFab m_cutcellmask; bool m_allregular = false; bool m_ok = false; @@ -105,6 +104,7 @@ protected: private: template friend class GShopLevel; + // Need this function to work around a gcc bug. void setRegularLevel () { m_allregular = true; @@ -296,12 +296,12 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, m_bndryarea.define(m_grids, m_dmap, 1, ng, mf_info); m_bndrycent.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); m_bndrynorm.define(m_grids, m_dmap, AMREX_SPACEDIM, ng, mf_info); - + m_multicut_fcx.define(m_grids, m_dmap, 1, ng, mf_info); + m_multicut_fcy.define(m_grids, m_dmap, 1, ng, mf_info); + m_multicut_fcz.define(m_grids, m_dmap, 1, ng, mf_info); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { m_areafrac[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, 1, ng, mf_info); - multicut_face[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), - m_dmap, 1, ng, mf_info); m_facecent[idim].define(amrex::convert(m_grids, IntVect::TheDimensionVector(idim)), m_dmap, AMREX_SPACEDIM-1, ng, mf_info); IntVect edge_type{1}; edge_type[idim] = 0; @@ -363,7 +363,6 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Array4 const& bar = m_bndryarea.array(mfi); Array4 const& bct = m_bndrycent.array(mfi); Array4 const& bnm = m_bndrynorm.array(mfi); - AMREX_D_TERM(Array4 const& apx = m_areafrac[0].array(mfi);, Array4 const& apy = m_areafrac[1].array(mfi);, Array4 const& apz = m_areafrac[2].array(mfi);); @@ -423,13 +422,14 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, Array4 const& ym2 = M2[1].array(); Array4 const& zm2 = M2[2].array(); - AMREX_D_TERM(Array4 const& mcx = multicut_face[0].array(mfi);, - Array4 const& mcy = multicut_face[1].array(mfi);, - Array4 const& mcz = multicut_face[2].array(mfi);); + Array4 const& mcx = m_multicut_fcx.array(mfi); + Array4 const& mcy = m_multicut_fcy.array(mfi); + Array4 const& mcz = m_multicut_fcz.array(mfi); nmc = build_faces(vbx, cfg, ftx, fty, ftz, xdg, ydg, zdg, lst, xip, yip, zip, apx, apy, apz, fcx, fcy, fcz, - xm2, ym2, zm2, dx, problo, cover_multiple_cuts, plt_multiple_cuts, mcx, mcy, mcz); + xm2, ym2, zm2, dx, problo, cover_multiple_cuts, + plt_multiple_cuts, mcx, mcy, mcz); cellflagtmp.resize(m_cellflag[mfi].box()); Elixir cellflagtmp_eli = cellflagtmp.elixir(); @@ -438,7 +438,8 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, build_cells(vbx, cfg, ftx, fty, ftz, apx, apy, apz, fcx, fcy, fcz, xm2, ym2, zm2, dx, vfr, ctr, bar, bct, bnm, cfgtmp, lst, - small_volfrac, geom, extend_domain_face, cover_multiple_cuts, plt_multiple_cuts, + small_volfrac, geom, extend_domain_face, + cover_multiple_cuts, plt_multiple_cuts, nsm, nmc); // Because it is used in a synchronous reduction kernel in @@ -480,10 +481,11 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, clst, geom); } - Array4 const& mcx = multicut_face[0].array(mfi); + Array4 const& mcx = m_multicut_fcx.array(mfi); nmc = build_faces(vbx, cfg, ftx, fty, lst, xip, yip, apx, apy, fcx, fcy, - dx, problo, cover_multiple_cuts, nsm, plt_multiple_cuts, mcx); + dx, problo, cover_multiple_cuts, nsm, + plt_multiple_cuts, mcx); build_cells(vbx, cfg, ftx, fty, apx, apy, dx, vfr, ctr, bar, bct, bnm, lst, small_volfrac, geom, extend_domain_face, @@ -495,29 +497,29 @@ GShopLevel::define_fine (G const& gshop, const Geometry& geom, } ParallelAllReduce::Sum({nsmallcells,nmulticuts}, ParallelContext::CommunicatorSub()); + if (plt_multiple_cuts && nmulticuts > 0) { amrex::Print() << "Total number of multicuts = " << nmulticuts << "\n"; amrex::Print() << "Plotting multicut locations..." << "\n"; - amrex::Print() << "Max is = " << multicut_face[0].max(0) << "\n"; #if (AMREX_SPACEDIM == 3) - if (multicut_face[0].max(0) > 2) + if (m_multicut_fcx.max(0) > 2) { - WriteSingleLevelPlotfile("plt.multicut.x", multicut_face[0], {"multicut_fcx"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.multicut.x", m_multicut_fcx, {"multicut_fcx"}, geom, 0.0, 0); } - if (multicut_face[1].max(0) > 2) + if (m_multicut_fcy.max(0) > 2) { - WriteSingleLevelPlotfile("plt.multicut.y", multicut_face[1], {"multicut_fcy"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.multicut.y", m_multicut_fcy, {"multicut_fcy"}, geom, 0.0, 0); } - if (multicut_face[2].max(0) > 2) + if (m_multicut_fcz.max(0) > 2) { - WriteSingleLevelPlotfile("plt.multicut.z", multicut_face[2], {"multicut_fcz"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.multicut.z", m_multicut_fcz, {"multicut_fcz"}, geom, 0.0, 0); } #elif (AMREX_SPACEDIM == 2) - if (multicut_face[0].max(0) > 2) + if (m_multicut_fcx.max(0) > 2) { - WriteSingleLevelPlotfile("plt.multicut", multicut_face[0], {"multicut_fcx"}, geom, 0.0, 0); + WriteSingleLevelPlotfile("plt.multicut", m_multicut_fcx, {"multicut_fcx"}, geom, 0.0, 0); } #endif amrex::Abort("amrex::EB2:: more than 2 cuts not supported"); @@ -621,4 +623,4 @@ GShopLevel::GShopLevel (IndexSpace const* is, int /*ilev*/, int max_grid_size } -#endif +#endif \ No newline at end of file