From f3935bea752e467f7728bfcd8cc8948bb0c5205a Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Thu, 22 May 2025 16:12:58 -0400 Subject: [PATCH 001/139] saving flux squared for standard deviation --- .../pumipic_particle_data_structure.cpp | 1473 +++++++++-------- test/test_pumi_tally_impl_methods.cpp | 790 +++++---- 2 files changed, 1210 insertions(+), 1053 deletions(-) diff --git a/src/pumitallyopenmc/pumipic_particle_data_structure.cpp b/src/pumitallyopenmc/pumipic_particle_data_structure.cpp index 1a2b9f5..edf3525 100644 --- a/src/pumitallyopenmc/pumipic_particle_data_structure.cpp +++ b/src/pumitallyopenmc/pumipic_particle_data_structure.cpp @@ -3,736 +3,837 @@ // #include "pumipic_particle_data_structure.h" -#include -#include #include -#include +#include +#include +#include #include #include -#include +#include #include -#include +#include #include namespace pumiinopenmc { - struct TallyTimes { - double initialization_time = 0.0; - double total_time_to_tally = 0.0; - double vtk_file_write_time = 0.0; - - void print_times() const { - printf("\n"); - printf("[TIME] Initialization time : %f seconds\n", initialization_time); - printf("[TIME] Total time to tally : %f seconds\n", total_time_to_tally); - printf("[TIME] VTK file write time : %f seconds\n", vtk_file_write_time); - printf("[TIME] Total PumiPic time : %f seconds\n", - initialization_time + total_time_to_tally + vtk_file_write_time); - } - }; - - // ------------------------------------------------------------------------------------------------// - // * Data structure for PumiPic - // Particle: 0-origin, 1-destination, 2-particle_id, 3-in_advance_particle_queue, 4-weight - typedef pumipic::MemberTypes PPParticle; - typedef pumipic::ParticleStructure PPPS; - typedef Kokkos::DefaultExecutionSpace PPExeSpace; - - // ------------------------------------------------------------------------------------------------// - // * Helper Functions - [[deprecated("Use move_to_next_element which is appropriate for new search class in pumipic. [!Note] It my show this even though it is not used due to template instantiation.")]] - void pp_move_to_new_element(Omega_h::Mesh &mesh, PPPS *ptcls, Omega_h::Write &elem_ids, - Omega_h::Write &ptcl_done, - Omega_h::Write &lastExit); - - std::unique_ptr pp_create_particle_structure(Omega_h::Mesh mesh, pumipic::lid_t numPtcls); - - void start_pumi_particles_in_0th_element(Omega_h::Mesh &mesh, pumiinopenmc::PPPS *ptcls); - - PumiTally::~PumiTally() { - pimpl.reset(nullptr); - Kokkos::finalize(); - } - - // ------------------------------------------------------------------------------------------------// - // * Struct for PumiParticleAtElemBoundary - struct PumiParticleAtElemBoundary { - PumiParticleAtElemBoundary(size_t nelems, size_t capacity); - - void operator()(Omega_h::Mesh &mesh, pumiinopenmc::PPPS *ptcls, Omega_h::Write &elem_ids, - Omega_h::Write &next_elems, - Omega_h::Write &inter_faces, Omega_h::Write &lastExit, - Omega_h::Write &inter_points, Omega_h::Write &ptcl_done, - typeof(ptcls->get<0>()) origin_segment, typeof(ptcls->get<1>()) dest_segment); - - void updatePrevXPoint(Omega_h::Write &xpoints); - - void updatePrevXPoint(PPPS *ptcls); +struct TallyTimes { + double initialization_time = 0.0; + double total_time_to_tally = 0.0; + double vtk_file_write_time = 0.0; + + void print_times() const { + printf("\n"); + printf("[TIME] Initialization time : %f seconds\n", + initialization_time); + printf("[TIME] Total time to tally : %f seconds\n", + total_time_to_tally); + printf("[TIME] VTK file write time : %f seconds\n", + vtk_file_write_time); + printf("[TIME] Total PumiPic time : %f seconds\n", + initialization_time + total_time_to_tally + vtk_file_write_time); + } +}; + +// ------------------------------------------------------------------------------------------------// +// * Data structure for PumiPic +// Particle: 0-origin, 1-destination, 2-particle_id, +// 3-in_advance_particle_queue, 4-weight +typedef pumipic::MemberTypes + PPParticle; +typedef pumipic::ParticleStructure PPPS; +typedef Kokkos::DefaultExecutionSpace PPExeSpace; + +// ------------------------------------------------------------------------------------------------// +// * Helper Functions +[[deprecated("Use move_to_next_element which is appropriate for new search " + "class in pumipic. [!Note] It my show this even though it is not " + "used due to template instantiation.")]] +void pp_move_to_new_element(Omega_h::Mesh &mesh, PPPS *ptcls, + Omega_h::Write &elem_ids, + Omega_h::Write &ptcl_done, + Omega_h::Write &lastExit); + +std::unique_ptr pp_create_particle_structure(Omega_h::Mesh mesh, + pumipic::lid_t numPtcls); + +void start_pumi_particles_in_0th_element(Omega_h::Mesh &mesh, + pumiinopenmc::PPPS *ptcls); + +PumiTally::~PumiTally() { + pimpl.reset(nullptr); + Kokkos::finalize(); +} + +// ------------------------------------------------------------------------------------------------// +// * Struct for PumiParticleAtElemBoundary +struct PumiParticleAtElemBoundary { + PumiParticleAtElemBoundary(size_t nelems, size_t capacity); - void evaluateFlux(PPPS *ptcls, Omega_h::Write xpoints, Omega_h::Write elem_ids, - Omega_h::Write ptcl_done); + void operator()(Omega_h::Mesh &mesh, pumiinopenmc::PPPS *ptcls, + Omega_h::Write &elem_ids, + Omega_h::Write &next_elems, + Omega_h::Write &inter_faces, + Omega_h::Write &lastExit, + Omega_h::Write &inter_points, + Omega_h::Write &ptcl_done, + typeof(ptcls->get<0>()) origin_segment, + typeof(ptcls->get<1>()) dest_segment); + + void updatePrevXPoint(Omega_h::Write &xpoints); + + void updatePrevXPoint(PPPS *ptcls); + + void evaluateFlux(PPPS *ptcls, Omega_h::Write xpoints, + Omega_h::Write elem_ids, + Omega_h::Write ptcl_done); + + void finalizeAndWritePumiFlux(Omega_h::Mesh &full_mesh, + const std::string &filename); - void finalizeAndWritePumiFlux(Omega_h::Mesh &full_mesh, const std::string &filename); + Omega_h::Reals normalizeFlux(Omega_h::Mesh &mesh); + + void mark_initial_as(bool initial); + + void compute_total_tracklength(PPPS *ptcls); + void initialize_flux_array(size_t nelems, size_t nEgroups); + + bool initial_; // in initial run, flux is not tallied + // Dims: nelems, nEgroups, 2 + Kokkos::View flux_; + Omega_h::Write prev_xpoint_; + Omega_h::Write total_tracklength_; +}; +// ------------------------------------------------------------------------------------------------// + +// ------------------------------------------------------------------------------------------------// +// * Struct for PumiTallyImpl +struct PumiTallyImpl { + int64_t pumi_ps_size = 1000000; // hundred thousand + std::string oh_mesh_fname; + + Omega_h::Library oh_lib; + Omega_h::Mesh full_mesh_; + + std::unique_ptr pp_lib = nullptr; + std::unique_ptr p_picparts_ = nullptr; + std::unique_ptr pumipic_ptcls = nullptr; + + long double pumipic_tol = 1e-8; + bool is_pumipic_initialized = false; + int64_t iter_count_ = 0; + double total_initial_weight_ = 0.0; + + std::unique_ptr + p_pumi_particle_at_elem_boundary_handler; + std::unique_ptr< + ParticleTracer> + p_particle_tracer_; - Omega_h::Reals normalizeFlux(Omega_h::Mesh &mesh); + Omega_h::Write device_pos_buffer_; + Omega_h::Write device_in_adv_que_; + Omega_h::Write weights_; - void mark_initial_as(bool initial); + TallyTimes tally_times; - void compute_total_tracklength(PPPS *ptcls); + // * Constructor + PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, int &argc, + char **&argv); // fixme extra & - bool initial_; // in initial run, flux is not tallied - Omega_h::Write flux_; - Omega_h::Write prev_xpoint_; - Omega_h::Write total_tracklength_; - }; - // ------------------------------------------------------------------------------------------------// + // * Destructor + ~PumiTallyImpl() = default; - // ------------------------------------------------------------------------------------------------// - // * Struct for PumiTallyImpl - struct PumiTallyImpl { - int64_t pumi_ps_size = 1000000; // hundred thousand - std::string oh_mesh_fname; + // Functions + void create_and_initialize_pumi_particle_structure(Omega_h::Mesh *mesh); - Omega_h::Library oh_lib; - Omega_h::Mesh full_mesh_; + void load_pumipic_mesh_and_init_particles(int &argc, char **&argv); - std::unique_ptr pp_lib = nullptr; - std::unique_ptr p_picparts_ = nullptr; - std::unique_ptr pumipic_ptcls = nullptr; + Omega_h::Mesh *partition_pumipic_mesh(); - long double pumipic_tol = 1e-8; - bool is_pumipic_initialized = false; - int64_t iter_count_ = 0; - double total_initial_weight_ = 0.0; + void init_pumi_libs(int &argc, char **&argv); - std::unique_ptr p_pumi_particle_at_elem_boundary_handler; - std::unique_ptr> p_particle_tracer_; + void search_and_rebuild(bool initial, bool migrate = true); - Omega_h::Write device_pos_buffer_; - Omega_h::Write device_in_adv_que_; - Omega_h::Write weights_; + void read_pumipic_lib_and_full_mesh(int &argc, char **&argv); - TallyTimes tally_times; + void initialize_particle_location(double *init_particle_positions, + int64_t size); - // * Constructor - PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, int &argc, char **&argv); // fixme extra & + void move_to_next_location(double *particle_destinations, int8_t *flying, + double *weights, int64_t size); - // * Destructor - ~PumiTallyImpl() = default; + void write_pumi_tally_mesh(); - // Functions - void create_and_initialize_pumi_particle_structure(Omega_h::Mesh *mesh); + void copy_data_to_device(double *init_particle_positions); - void load_pumipic_mesh_and_init_particles(int &argc, char **&argv); + void search_initial_elements(); - Omega_h::Mesh *partition_pumipic_mesh(); + void copy_and_reset_flying_flag(int8_t *flying); - void init_pumi_libs(int &argc, char **&argv); + void copy_weights(double *weights); +}; - void search_and_rebuild(bool initial, bool migrate = true); +PumiTallyImpl::PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, + int &argc, char **&argv) { + pumi_ps_size = num_particles; + oh_mesh_fname = mesh_filename; - void read_pumipic_lib_and_full_mesh(int &argc, char **&argv); + device_pos_buffer_ = + Omega_h::Write(pumi_ps_size * 3, 0.0, "device_pos_buffer"); + device_in_adv_que_ = + Omega_h::Write(pumi_ps_size, 0, "device_in_adv_que"); + weights_ = Omega_h::Write(pumi_ps_size, 0.0, "weights"); - void initialize_particle_location(double *init_particle_positions, int64_t size); + // todo can track lengths be here? - void move_to_next_location(double *particle_destinations, int8_t *flying, double *weights, int64_t size); + load_pumipic_mesh_and_init_particles(argc, argv); + start_pumi_particles_in_0th_element(*p_picparts_->mesh(), + pumipic_ptcls.get()); - void write_pumi_tally_mesh(); + p_particle_tracer_ = std::make_unique< + ParticleTracer>( + *p_picparts_, pumipic_ptcls.get(), + *p_pumi_particle_at_elem_boundary_handler, 1e-8); +} - void copy_data_to_device(double *init_particle_positions); - - void search_initial_elements(); - - void copy_and_reset_flying_flag(int8_t *flying); - - void copy_weights(double *weights); - }; - - PumiTallyImpl::PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, int &argc, char **&argv) { - pumi_ps_size = num_particles; - oh_mesh_fname = mesh_filename; - - device_pos_buffer_ = Omega_h::Write(pumi_ps_size * 3, 0.0, "device_pos_buffer"); - device_in_adv_que_ = Omega_h::Write(pumi_ps_size, 0, "device_in_adv_que"); - weights_ = Omega_h::Write(pumi_ps_size, 0.0, "weights"); - - // todo can track lengths be here? - - load_pumipic_mesh_and_init_particles(argc, argv); - start_pumi_particles_in_0th_element(*p_picparts_->mesh(), pumipic_ptcls.get()); - - p_particle_tracer_ = std::make_unique>(*p_picparts_, pumipic_ptcls.get(), *p_pumi_particle_at_elem_boundary_handler, 1e-8); - } - - void PumiTallyImpl::initialize_particle_location(double *init_particle_positions, int64_t size) { - // copy to host buffer - assert(size == pumi_ps_size * 3); - copy_data_to_device(init_particle_positions); - search_initial_elements(); - // TODO Get total initial particle weight +void PumiTallyImpl::initialize_particle_location( + double *init_particle_positions, int64_t size) { + // copy to host buffer + assert(size == pumi_ps_size * 3); + copy_data_to_device(init_particle_positions); + search_initial_elements(); + // TODO Get total initial particle weight #ifdef PUMI_MEASURE_TIME - Kokkos::fence(); + Kokkos::fence(); #endif - } - - void - PumiTallyImpl::move_to_next_location(double *particle_destinations, int8_t *flying, double *weights, int64_t size) { - assert(size == pumi_ps_size * 3); - - // copy to device buffer - copy_data_to_device(particle_destinations); - // copy fly to device buffer - copy_and_reset_flying_flag(flying); - copy_weights(weights); - - // copy position buffer ps - auto particle_dest = pumipic_ptcls->get<1>(); - auto in_flight = pumipic_ptcls->get<3>(); - - int64_t pumi_ps_size_ = pumi_ps_size; - const auto &device_pos_buffer_l = device_pos_buffer_; - const auto &device_in_adv_que_l = device_in_adv_que_; - - auto set_particle_dest = PS_LAMBDA(const int &e, const int &pid, const int &mask) { - if (mask > 0 && pid < pumi_ps_size_) { - particle_dest(pid, 0) = device_pos_buffer_l[pid * 3 + 0]; - particle_dest(pid, 1) = device_pos_buffer_l[pid * 3 + 1]; - particle_dest(pid, 2) = device_pos_buffer_l[pid * 3 + 2]; - - // everyone is in flight for this initial search - in_flight(pid) = device_in_adv_que_l[pid]; - } - }; - pumipic::parallel_for(pumipic_ptcls.get(), set_particle_dest, "set particle position as dest"); - - bool migrate = iter_count_ % 100 == 0; - iter_count_++; - search_and_rebuild(false, migrate); +} + +void PumiTallyImpl::move_to_next_location(double *particle_destinations, + int8_t *flying, double *weights, + int64_t size) { + assert(size == pumi_ps_size * 3); + + // copy to device buffer + copy_data_to_device(particle_destinations); + // copy fly to device buffer + copy_and_reset_flying_flag(flying); + copy_weights(weights); + + // copy position buffer ps + auto particle_dest = pumipic_ptcls->get<1>(); + auto in_flight = pumipic_ptcls->get<3>(); + + int64_t pumi_ps_size_ = pumi_ps_size; + const auto &device_pos_buffer_l = device_pos_buffer_; + const auto &device_in_adv_que_l = device_in_adv_que_; + + auto set_particle_dest = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if (mask > 0 && pid < pumi_ps_size_) { + particle_dest(pid, 0) = device_pos_buffer_l[pid * 3 + 0]; + particle_dest(pid, 1) = device_pos_buffer_l[pid * 3 + 1]; + particle_dest(pid, 2) = device_pos_buffer_l[pid * 3 + 2]; + + // everyone is in flight for this initial search + in_flight(pid) = device_in_adv_que_l[pid]; + } + }; + pumipic::parallel_for(pumipic_ptcls.get(), set_particle_dest, + "set particle position as dest"); + + bool migrate = iter_count_ % 100 == 0; + iter_count_++; + search_and_rebuild(false, migrate); #ifdef PUMI_MEASURE_TIME - Kokkos::fence(); + Kokkos::fence(); #endif - } +} - void PumiTallyImpl::write_pumi_tally_mesh() { - p_pumi_particle_at_elem_boundary_handler->finalizeAndWritePumiFlux(full_mesh_, "fluxresult.vtk"); +void PumiTallyImpl::write_pumi_tally_mesh() { + p_pumi_particle_at_elem_boundary_handler->finalizeAndWritePumiFlux( + full_mesh_, "fluxresult.vtk"); #ifdef PUMI_MEASURE_TIME - Kokkos::fence(); + Kokkos::fence(); #endif - } - - void PumiTallyImpl::copy_and_reset_flying_flag(int8_t *flying) { - // todo get the size too - auto device_in_adv_que_l = device_in_adv_que_; - Kokkos::View > - host_flying_view(flying, pumi_ps_size); - Kokkos::View > - device_flying_view(device_in_adv_que_l.data(), device_in_adv_que_l.size()); - Kokkos::deep_copy(device_flying_view, host_flying_view); - - for (int64_t pid = 0; pid < pumi_ps_size; ++pid) { - // reset flying flag to zero - flying[pid] = 0; - } - } - - void PumiTallyImpl::copy_weights(double *weights) { - auto weights_l = weights_; - Kokkos::View > - host_weights_view(weights, pumi_ps_size); - Kokkos::View > - device_weights_view(weights_l.data(), weights_l.size()); - - Kokkos::deep_copy(device_weights_view, host_weights_view); - - auto p_wgt = pumipic_ptcls->get<4>(); - auto copy_particle_weights = PS_LAMBDA( - const int &e, - const int &pid, - const int &mask) { - p_wgt(pid) = weights_l[pid]; - }; - pumipic::parallel_for(pumipic_ptcls.get(), copy_particle_weights, "copy particle weights"); - } - - void PumiTallyImpl::search_initial_elements() {// assign the location to ptcl dest - auto particle_dest = pumipic_ptcls->get<1>(); - auto in_flight = pumipic_ptcls->get<3>(); - - int64_t pumi_ps_size_l = pumi_ps_size; - const auto &device_pos_buffer_l = device_pos_buffer_; - - auto set_particle_dest = PS_LAMBDA(const int &e, const int &pid, const int &mask) { - if (mask > 0 && pid < pumi_ps_size_l) { - particle_dest(pid, 0) = device_pos_buffer_l[pid * 3 + 0]; - particle_dest(pid, 1) = device_pos_buffer_l[pid * 3 + 1]; - particle_dest(pid, 2) = device_pos_buffer_l[pid * 3 + 2]; - - // everyone is in flight for this initial search - in_flight(pid) = 1; - } - }; - pumipic::parallel_for(pumipic_ptcls.get(), set_particle_dest, "set initial position as dest"); - - // *initial* build and search to find the initial elements of the particles - search_and_rebuild(true, true); - is_pumipic_initialized = true; - } - - void PumiTallyImpl::copy_data_to_device(double *init_particle_positions) { - // fixme it should get size too to avoid memory error - Kokkos::View > - host_pos_view(init_particle_positions, pumi_ps_size * 3); - - Kokkos::View > - device_pos_view(device_pos_buffer_.data(), pumi_ps_size * 3); - - Kokkos::deep_copy(device_pos_view, host_pos_view); - } - - - // methods for PumiTallyImpl and PumiParticleAtElemBoundary - - void PumiTallyImpl::init_pumi_libs(int &argc, char **&argv) { - pp_lib = std::make_unique(&argc, &argv); - oh_lib = pp_lib->omega_h_lib(); - } - - [[deprecated("Use move_to_next_element which is appropriate for new search class in pumipic. [!Note] It my show this even though it is not used due to template instantiation.")]] - void pp_move_to_new_element(Omega_h::Mesh &mesh, PPPS *ptcls, Omega_h::Write &elem_ids, - Omega_h::Write &ptcl_done, - Omega_h::Write &lastExit) { - const int dim = mesh.dim(); - const auto &face2elems = mesh.ask_up(dim - 1, dim); - const auto &face2elemElem = face2elems.ab2b; - const auto &face2elemOffset = face2elems.a2ab; - const auto in_flight = ptcls->get<3>(); - - auto set_next_element = - PS_LAMBDA(const int &e, const int &pid, const int &mask) { - if (mask > 0 && !ptcl_done[pid] && in_flight(pid)) { - auto searchElm = elem_ids[pid]; - auto bridge = lastExit[pid]; - auto e2f_first = face2elemOffset[bridge]; - auto e2f_last = face2elemOffset[bridge + 1]; - auto upFaces = e2f_last - e2f_first; - assert(upFaces == 2); - auto faceA = face2elemElem[e2f_first]; - auto faceB = face2elemElem[e2f_first + 1]; - assert(faceA != faceB); - assert(faceA == searchElm || faceB == searchElm); - auto nextElm = (faceA == searchElm) ? faceB : faceA; - elem_ids[pid] = nextElm; - } - }; - parallel_for(ptcls, set_next_element, "pumipic_set_next_element"); - } - - void move_to_next_element(PPPS *ptcls, Omega_h::Write &elem_ids, Omega_h::Write &next_elems) { - auto in_flight = ptcls->get<3>(); - auto move_to_next = PS_LAMBDA(const int e, const int pid, const int mask) { - // move only if particle in flight and not leaving the domain - if (mask > 0 && in_flight(pid) && next_elems[pid] != -1) { - elem_ids[pid] = next_elems[pid]; - } - }; - pumipic::parallel_for(ptcls, move_to_next, "move to next element"); - } - - void apply_boundary_condition(Omega_h::Mesh &mesh, PPPS *ptcls, - Omega_h::Write &elem_ids, - Omega_h::Write &next_elems, - Omega_h::Write &ptcl_done, - Omega_h::Write &lastExit, - Omega_h::Write &xFace, - Omega_h::Write &inter_points) { - - // TODO: make this a member variable of the struct - auto particle_destination = ptcls->get<1>(); - auto checkExposedEdges = - PS_LAMBDA(const int e, const int pid, const int mask) { - if (mask > 0 && !ptcl_done[pid]) { - bool reached_destination = (lastExit[pid] == -1); - bool hit_boundary = ((next_elems[pid] == -1) && (elem_ids[pid] != -1)); - ptcl_done[pid] = (reached_destination || hit_boundary) ? 1 : ptcl_done[pid]; - - if (hit_boundary) { // just reached the boundary - xFace[pid] = lastExit[pid]; - // particle reaches the boundary - particle_destination(pid, 0) = inter_points[pid * 3]; - particle_destination(pid, 1) = inter_points[pid * 3 + 1]; - particle_destination(pid, 2) = inter_points[pid * 3 + 2]; - } - } - }; - pumipic::parallel_for(ptcls, checkExposedEdges, "apply vacumm boundary condition"); - } - - PumiParticleAtElemBoundary::PumiParticleAtElemBoundary(size_t nelems, size_t capacity) - : flux_(nelems, 0.0, "flux"), - prev_xpoint_(capacity * 3, 0.0, "prev_xpoint"), - total_tracklength_(capacity, 0.0, "total_tracklength"), - initial_(true) { - printf( - "[INFO] Particle handler at boundary with %d elements and %d " - "x points size (3 * n_particles)\n", - flux_.size(), prev_xpoint_.size()); - } - - void PumiParticleAtElemBoundary::operator()(Omega_h::Mesh &mesh, pumiinopenmc::PPPS *ptcls, - Omega_h::Write &elem_ids, - Omega_h::Write &next_elems, - Omega_h::Write &inter_faces, - Omega_h::Write &lastExit, - Omega_h::Write &inter_points, - Omega_h::Write &ptcl_done, - typeof(ptcls->get<0>()) origin_segment, - typeof(ptcls->get<1>()) dest_segment) { - if (!initial_) { - evaluateFlux(ptcls, inter_points, elem_ids, ptcl_done); - updatePrevXPoint(inter_points); - } - apply_boundary_condition(mesh, ptcls, elem_ids, next_elems, ptcl_done, lastExit, inter_faces, inter_points); - move_to_next_element(ptcls, elem_ids, next_elems); - } - - void PumiParticleAtElemBoundary::mark_initial_as(bool initial) { - initial_ = initial; - } - - void PumiParticleAtElemBoundary::updatePrevXPoint(Omega_h::Write &xpoints) { - OMEGA_H_CHECK_PRINTF( - xpoints.size() <= prev_xpoint_.size() && prev_xpoint_.size() != 0, - "xpoints size %d is greater than prev_xpoint size %d\n", xpoints.size(), - prev_xpoint_.size()); - auto &prev_xpoint = prev_xpoint_; - auto update = OMEGA_H_LAMBDA(Omega_h::LO - i) { prev_xpoint[i] = xpoints[i]; }; - Omega_h::parallel_for(xpoints.size(), update, "update previous xpoints"); - } - - void PumiParticleAtElemBoundary::updatePrevXPoint(PPPS *ptcls) { - // todo add checks of size - auto prev_xpoints_l = prev_xpoint_; - OMEGA_H_CHECK_PRINTF(ptcls->capacity() * 3 == prev_xpoints_l.size(), - "Error: prev_xpoints_s are not size properly capacity %d size %d\n", ptcls->capacity(), - prev_xpoints_l.size()); - auto xpoints = ptcls->get<0>(); - auto update = PS_LAMBDA( - const auto &e, - const auto &pid, - const auto &mask) { - prev_xpoints_l[pid * 3 + 0] = xpoints(pid, 0); - prev_xpoints_l[pid * 3 + 1] = xpoints(pid, 1); - prev_xpoints_l[pid * 3 + 2] = xpoints(pid, 2); - }; - pumipic::parallel_for(ptcls, update, "update previous xpoints from origin points"); - } - - void PumiParticleAtElemBoundary::evaluateFlux(PPPS *ptcls, Omega_h::Write xpoints, - Omega_h::Write elem_ids, - Omega_h::Write ptcl_done) { - //Omega_h::Real total_particles = ptcls->nPtcls(); - auto prev_xpoint = prev_xpoint_; - auto flux = flux_; - auto total_tracklength_l = total_tracklength_; - auto in_flight = ptcls->get<3>(); - auto p_wgt = ptcls->get<4>(); - auto xpoints_l = xpoints; // todo shouldn't need it, so remove - - auto evaluate_flux = - PS_LAMBDA( - const int &e, - const int &pid, - const int &mask) { - if ((mask > 0) && (in_flight(pid) == 1) && !ptcl_done[pid]) { - OMEGA_H_CHECK_PRINTF(total_tracklength_l[pid] >= 0.0, - "ERROR: Particle is moving but the tracklength is negative: %.16f\n", - total_tracklength_l[pid]); - - Omega_h::Vector<3> dest = {xpoints_l[pid * 3 + 0], - xpoints_l[pid * 3 + 1], - xpoints_l[pid * 3 + 2]}; - Omega_h::Vector<3> orig = {prev_xpoint[pid * 3 + 0], - prev_xpoint[pid * 3 + 1], - prev_xpoint[pid * 3 + 2]}; - - Omega_h::Real segment_length = Omega_h::norm(dest - orig); // / total_particles; - if (segment_length > total_tracklength_l[pid] + - 1e-6) { // tol for float operations and search algorithm's inaccuracy - // fixme: something wrong with orig: previous_xpoints are incorrect at least for the first run - printf("ERROR: Segment length in an element cannot be greater than the total tracklength but found %.16f, %.16f of pid %d crossing el %d starting at %d\nOrig: (%.16f, %.16f, %.16f), Dest: (%.16f, %.16f, %.16f)\n", - segment_length, total_tracklength_l[pid], pid, elem_ids[pid], e, - orig[0], orig[1], orig[2], dest[0], dest[1], dest[2]); - } - - Omega_h::Real contribution = segment_length * p_wgt(pid); - - Kokkos::atomic_add(&flux[elem_ids[pid]], contribution); - } - }; - pumipic::parallel_for(ptcls, evaluate_flux, "flux evaluation loop"); - } - - Omega_h::Reals PumiParticleAtElemBoundary::normalizeFlux(Omega_h::Mesh &mesh) { - const Omega_h::LO nelems = mesh.nelems(); - const auto &el2n = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; - const auto &coords = mesh.coords(); - - auto flux = flux_; - auto total_tracklength_l = total_tracklength_; - - Omega_h::Write tet_volumes(flux_.size(), -1.0, "tet_volumes"); - Omega_h::Write normalized_flux(flux_.size(), -1.0, "normalized flux"); - - auto normalize_flux_with_volume = OMEGA_H_LAMBDA(Omega_h::LO - elem_id) { - const auto elem_verts = Omega_h::gather_verts<4>(el2n, elem_id); - const auto elem_vert_coords = Omega_h::gather_vectors<4, 3>(coords, elem_verts); - - auto b = Omega_h::simplex_basis<3, 3>(elem_vert_coords); - auto volume = Omega_h::simplex_size_from_basis(b); - - tet_volumes[elem_id] = volume; - normalized_flux[elem_id] = flux[elem_id] / (volume * total_tracklength_l.size()); - }; - Omega_h::parallel_for(tet_volumes.size(), normalize_flux_with_volume, - "normalize flux"); - - mesh.add_tag(Omega_h::REGION, "volume", 1, Omega_h::Reals(tet_volumes)); - return {normalized_flux}; - } - - void PumiParticleAtElemBoundary::finalizeAndWritePumiFlux(Omega_h::Mesh &full_mesh, const std::string &filename) { - const auto &normalized_flux = normalizeFlux(full_mesh); - full_mesh.add_tag(Omega_h::REGION, "flux", 1, normalized_flux); - Omega_h::vtk::write_parallel(filename, &full_mesh, 3); - } - - void pumiUpdatePtclPositions(PPPS *ptcls) { - auto x_ps_d = ptcls->get<0>(); - auto xtgt_ps_d = ptcls->get<1>(); - auto updatePtclPos = PS_LAMBDA(const int &, const int &pid, const bool &) { - x_ps_d(pid, 0) = xtgt_ps_d(pid, 0); - x_ps_d(pid, 1) = xtgt_ps_d(pid, 1); - x_ps_d(pid, 2) = xtgt_ps_d(pid, 2); - xtgt_ps_d(pid, 0) = 0.0; - xtgt_ps_d(pid, 1) = 0.0; - xtgt_ps_d(pid, 2) = 0.0; - }; - ps::parallel_for(ptcls, updatePtclPos); - } - - void PumiParticleAtElemBoundary::compute_total_tracklength(PPPS *ptcls) { - auto orig = ptcls->get<0>(); - auto dest = ptcls->get<1>(); - - auto total_tracklength_l = total_tracklength_; - - auto computeTrackLength = PS_LAMBDA( - const int &elemId, - const int &pid, - const bool &mask) { - Omega_h::Vector<3> p_orig = {orig(pid, 0), orig(pid, 1), orig(pid, 2)}; - Omega_h::Vector<3> p_dest = {dest(pid, 0), dest(pid, 1), dest(pid, 2)}; - Omega_h::Real track_length = Omega_h::norm(p_dest - p_orig); - total_tracklength_l[pid] = track_length; - }; - pumipic::parallel_for(ptcls, computeTrackLength, "compute total track length"); - } - - // search and update parent elements - //! @param initial initial search finds the initial location of the particles and doesn't tally - void PumiTallyImpl::search_and_rebuild(bool initial, const bool migrate) { - // initial cannot be false when is_pumipic_initialized is false - // may fail if simulated more than one batch - assert((is_pumipic_initialized == false && initial == true) || - (is_pumipic_initialized == true && initial == false)); - p_pumi_particle_at_elem_boundary_handler->mark_initial_as(initial); - auto orig = pumipic_ptcls->get<0>(); - auto dest = pumipic_ptcls->get<1>(); - auto pid = pumipic_ptcls->get<2>(); - - if (p_picparts_->mesh() == nullptr || p_picparts_->mesh()->nelems() == 0) { - fprintf(stderr, "ERROR: Mesh is empty\n"); - } - - // total tracklengths are used to calculate the flux - if (!initial) { - p_pumi_particle_at_elem_boundary_handler->updatePrevXPoint(pumipic_ptcls.get()); - p_pumi_particle_at_elem_boundary_handler->compute_total_tracklength(pumipic_ptcls.get()); - } - - bool isFoundAll = p_particle_tracer_->search(migrate); - - if (!isFoundAll) { - printf("ERROR: Not all particles are found. May need more loops in search\n"); - } - } - - - std::unique_ptr pp_create_particle_structure(Omega_h::Mesh mesh, pumipic::lid_t numPtcls) { - Omega_h::Int ne = mesh.nelems(); - pumiinopenmc::PPPS::kkLidView ptcls_per_elem("ptcls_per_elem", ne); - pumiinopenmc::PPPS::kkGidView element_gids("element_gids", ne); - - Kokkos::TeamPolicy policy; - - Omega_h::parallel_for( - ne, OMEGA_H_LAMBDA( - const Omega_h::LO &i) { element_gids(i) = i; }); - - Omega_h::parallel_for(mesh.nelems(), - OMEGA_H_LAMBDA(Omega_h::LO - id) { - ptcls_per_elem[id] = (id == 0) ? numPtcls : 0; - }); +} + +void PumiTallyImpl::copy_and_reset_flying_flag(int8_t *flying) { + // todo get the size too + auto device_in_adv_que_l = device_in_adv_que_; + Kokkos::View> + host_flying_view(flying, pumi_ps_size); + Kokkos::View> + device_flying_view(device_in_adv_que_l.data(), + device_in_adv_que_l.size()); + Kokkos::deep_copy(device_flying_view, host_flying_view); + + for (int64_t pid = 0; pid < pumi_ps_size; ++pid) { + // reset flying flag to zero + flying[pid] = 0; + } +} + +void PumiTallyImpl::copy_weights(double *weights) { + auto weights_l = weights_; + Kokkos::View> + host_weights_view(weights, pumi_ps_size); + Kokkos::View> + device_weights_view(weights_l.data(), weights_l.size()); + + Kokkos::deep_copy(device_weights_view, host_weights_view); + + auto p_wgt = pumipic_ptcls->get<4>(); + auto copy_particle_weights = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + p_wgt(pid) = weights_l[pid]; + }; + pumipic::parallel_for(pumipic_ptcls.get(), copy_particle_weights, + "copy particle weights"); +} + +void PumiTallyImpl::search_initial_elements() { // assign the location to ptcl + // dest + auto particle_dest = pumipic_ptcls->get<1>(); + auto in_flight = pumipic_ptcls->get<3>(); + + int64_t pumi_ps_size_l = pumi_ps_size; + const auto &device_pos_buffer_l = device_pos_buffer_; + + auto set_particle_dest = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if (mask > 0 && pid < pumi_ps_size_l) { + particle_dest(pid, 0) = device_pos_buffer_l[pid * 3 + 0]; + particle_dest(pid, 1) = device_pos_buffer_l[pid * 3 + 1]; + particle_dest(pid, 2) = device_pos_buffer_l[pid * 3 + 2]; + + // everyone is in flight for this initial search + in_flight(pid) = 1; + } + }; + pumipic::parallel_for(pumipic_ptcls.get(), set_particle_dest, + "set initial position as dest"); + + // *initial* build and search to find the initial elements of the particles + search_and_rebuild(true, true); + is_pumipic_initialized = true; +} + +void PumiTallyImpl::copy_data_to_device(double *init_particle_positions) { + // fixme it should get size too to avoid memory error + Kokkos::View> + host_pos_view(init_particle_positions, pumi_ps_size * 3); + + Kokkos::View> + device_pos_view(device_pos_buffer_.data(), pumi_ps_size * 3); + + Kokkos::deep_copy(device_pos_view, host_pos_view); +} + +// methods for PumiTallyImpl and PumiParticleAtElemBoundary + +void PumiTallyImpl::init_pumi_libs(int &argc, char **&argv) { + pp_lib = std::make_unique(&argc, &argv); + oh_lib = pp_lib->omega_h_lib(); +} + +[[deprecated("Use move_to_next_element which is appropriate for new search " + "class in pumipic. [!Note] It my show this even though it is not " + "used due to template instantiation.")]] +void pp_move_to_new_element(Omega_h::Mesh &mesh, PPPS *ptcls, + Omega_h::Write &elem_ids, + Omega_h::Write &ptcl_done, + Omega_h::Write &lastExit) { + const int dim = mesh.dim(); + const auto &face2elems = mesh.ask_up(dim - 1, dim); + const auto &face2elemElem = face2elems.ab2b; + const auto &face2elemOffset = face2elems.a2ab; + const auto in_flight = ptcls->get<3>(); + + auto set_next_element = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if (mask > 0 && !ptcl_done[pid] && in_flight(pid)) { + auto searchElm = elem_ids[pid]; + auto bridge = lastExit[pid]; + auto e2f_first = face2elemOffset[bridge]; + auto e2f_last = face2elemOffset[bridge + 1]; + auto upFaces = e2f_last - e2f_first; + assert(upFaces == 2); + auto faceA = face2elemElem[e2f_first]; + auto faceB = face2elemElem[e2f_first + 1]; + assert(faceA != faceB); + assert(faceA == searchElm || faceB == searchElm); + auto nextElm = (faceA == searchElm) ? faceB : faceA; + elem_ids[pid] = nextElm; + } + }; + parallel_for(ptcls, set_next_element, "pumipic_set_next_element"); +} + +void move_to_next_element(PPPS *ptcls, Omega_h::Write &elem_ids, + Omega_h::Write &next_elems) { + auto in_flight = ptcls->get<3>(); + auto move_to_next = PS_LAMBDA(const int e, const int pid, const int mask) { + // move only if particle in flight and not leaving the domain + if (mask > 0 && in_flight(pid) && next_elems[pid] != -1) { + elem_ids[pid] = next_elems[pid]; + } + }; + pumipic::parallel_for(ptcls, move_to_next, "move to next element"); +} + +void apply_boundary_condition(Omega_h::Mesh &mesh, PPPS *ptcls, + Omega_h::Write &elem_ids, + Omega_h::Write &next_elems, + Omega_h::Write &ptcl_done, + Omega_h::Write &lastExit, + Omega_h::Write &xFace, + Omega_h::Write &inter_points) { + + // TODO: make this a member variable of the struct + auto particle_destination = ptcls->get<1>(); + auto checkExposedEdges = + PS_LAMBDA(const int e, const int pid, const int mask) { + if (mask > 0 && !ptcl_done[pid]) { + bool reached_destination = (lastExit[pid] == -1); + bool hit_boundary = ((next_elems[pid] == -1) && (elem_ids[pid] != -1)); + ptcl_done[pid] = + (reached_destination || hit_boundary) ? 1 : ptcl_done[pid]; + + if (hit_boundary) { // just reached the boundary + xFace[pid] = lastExit[pid]; + // particle reaches the boundary + particle_destination(pid, 0) = inter_points[pid * 3]; + particle_destination(pid, 1) = inter_points[pid * 3 + 1]; + particle_destination(pid, 2) = inter_points[pid * 3 + 2]; + } + } + }; + pumipic::parallel_for(ptcls, checkExposedEdges, + "apply vacumm boundary condition"); +} + +void PumiParticleAtElemBoundary::initialize_flux_array(size_t nelems, + size_t nEgroups) { + Kokkos::resize(flux_, nelems, nEgroups, 2); + auto flux_l = flux_; + OMEGA_H_CHECK(flux_l.extent(0) == nelems); + OMEGA_H_CHECK(flux_l.extent(1) == nEgroups); + OMEGA_H_CHECK(flux_l.extent(2) == 2); +} + +PumiParticleAtElemBoundary::PumiParticleAtElemBoundary(size_t nelems, + size_t capacity) + : prev_xpoint_(capacity * 3, 0.0, "prev_xpoint"), + total_tracklength_(capacity, 0.0, "total_tracklength"), initial_(true) { + initialize_flux_array(nelems, 1); + printf("[INFO] Particle handler at boundary with %d elements and %d " + "x points size (3 * n_particles)\n", + flux_.size(), prev_xpoint_.size()); +} + +void PumiParticleAtElemBoundary::operator()( + Omega_h::Mesh &mesh, pumiinopenmc::PPPS *ptcls, + Omega_h::Write &elem_ids, + Omega_h::Write &next_elems, + Omega_h::Write &inter_faces, + Omega_h::Write &lastExit, + Omega_h::Write &inter_points, + Omega_h::Write &ptcl_done, + typeof(ptcls->get<0>()) origin_segment, + typeof(ptcls->get<1>()) dest_segment) { + if (!initial_) { + evaluateFlux(ptcls, inter_points, elem_ids, ptcl_done); + updatePrevXPoint(inter_points); + } + apply_boundary_condition(mesh, ptcls, elem_ids, next_elems, ptcl_done, + lastExit, inter_faces, inter_points); + move_to_next_element(ptcls, elem_ids, next_elems); +} + +void PumiParticleAtElemBoundary::mark_initial_as(bool initial) { + initial_ = initial; +} + +void PumiParticleAtElemBoundary::updatePrevXPoint( + Omega_h::Write &xpoints) { + OMEGA_H_CHECK_PRINTF(xpoints.size() <= prev_xpoint_.size() && + prev_xpoint_.size() != 0, + "xpoints size %d is greater than prev_xpoint size %d\n", + xpoints.size(), prev_xpoint_.size()); + auto &prev_xpoint = prev_xpoint_; + auto update = OMEGA_H_LAMBDA(Omega_h::LO i) { prev_xpoint[i] = xpoints[i]; }; + Omega_h::parallel_for(xpoints.size(), update, "update previous xpoints"); +} + +void PumiParticleAtElemBoundary::updatePrevXPoint(PPPS *ptcls) { + // todo add checks of size + auto prev_xpoints_l = prev_xpoint_; + OMEGA_H_CHECK_PRINTF( + ptcls->capacity() * 3 == prev_xpoints_l.size(), + "Error: prev_xpoints_s are not size properly capacity %d size %d\n", + ptcls->capacity(), prev_xpoints_l.size()); + auto xpoints = ptcls->get<0>(); + auto update = PS_LAMBDA(const auto &e, const auto &pid, const auto &mask) { + prev_xpoints_l[pid * 3 + 0] = xpoints(pid, 0); + prev_xpoints_l[pid * 3 + 1] = xpoints(pid, 1); + prev_xpoints_l[pid * 3 + 2] = xpoints(pid, 2); + }; + pumipic::parallel_for(ptcls, update, + "update previous xpoints from origin points"); +} + +void PumiParticleAtElemBoundary::evaluateFlux( + PPPS *ptcls, Omega_h::Write xpoints, + Omega_h::Write elem_ids, + Omega_h::Write ptcl_done) { + // Omega_h::Real total_particles = ptcls->nPtcls(); + auto prev_xpoint = prev_xpoint_; + auto flux = flux_; + auto total_tracklength_l = total_tracklength_; + auto in_flight = ptcls->get<3>(); + auto p_wgt = ptcls->get<4>(); + auto xpoints_l = xpoints; // todo shouldn't need it, so remove + + auto evaluate_flux = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if ((mask > 0) && (in_flight(pid) == 1) && !ptcl_done[pid]) { + OMEGA_H_CHECK_PRINTF( + total_tracklength_l[pid] >= 0.0, + "ERROR: Particle is moving but the tracklength is negative: %.16f\n", + total_tracklength_l[pid]); + + Omega_h::Vector<3> dest = {xpoints_l[pid * 3 + 0], xpoints_l[pid * 3 + 1], + xpoints_l[pid * 3 + 2]}; + Omega_h::Vector<3> orig = {prev_xpoint[pid * 3 + 0], + prev_xpoint[pid * 3 + 1], + prev_xpoint[pid * 3 + 2]}; + + Omega_h::Real segment_length = + Omega_h::norm(dest - orig); // / total_particles; + if (segment_length > + total_tracklength_l[pid] + 1e-6) { // tol for float operations and + // search algorithm's inaccuracy + // fixme: something wrong with orig: previous_xpoints are incorrect at + // least for the first run + printf("ERROR: Segment length in an element cannot be greater than the " + "total tracklength but found %.16f, %.16f of pid %d crossing el " + "%d starting at %d\nOrig: (%.16f, %.16f, %.16f), Dest: (%.16f, " + "%.16f, %.16f)\n", + segment_length, total_tracklength_l[pid], pid, elem_ids[pid], e, + orig[0], orig[1], orig[2], dest[0], dest[1], dest[2]); + } + + Omega_h::Real contribution = segment_length * p_wgt(pid); + + Kokkos::atomic_add(&flux(elem_ids[pid], 0, 0), contribution); + Kokkos::atomic_add(&flux(elem_ids[pid], 0, 1), + contribution * contribution); + } + }; + pumipic::parallel_for(ptcls, evaluate_flux, "flux evaluation loop"); +} + +Omega_h::Reals PumiParticleAtElemBoundary::normalizeFlux(Omega_h::Mesh &mesh) { + const Omega_h::LO nelems = mesh.nelems(); + const auto &el2n = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; + const auto &coords = mesh.coords(); + + auto flux = flux_; + auto total_tracklength_l = total_tracklength_; + + Omega_h::Write tet_volumes(flux_.extent(0), -1.0, + "tet_volumes"); + Omega_h::Write normalized_flux(flux_.extent(0), -1.0, + "normalized flux"); + Omega_h::Write normalized_flux_2(flux_.extent(0), -1.0, + "normalized flux squared"); + Omega_h::Write sd(flux_.extent(0), -1.0, "sd"); + + auto normalize_flux_with_volume = OMEGA_H_LAMBDA(Omega_h::LO elem_id) { + const auto elem_verts = Omega_h::gather_verts<4>(el2n, elem_id); + const auto elem_vert_coords = + Omega_h::gather_vectors<4, 3>(coords, elem_verts); + + auto b = Omega_h::simplex_basis<3, 3>(elem_vert_coords); + auto volume = Omega_h::simplex_size_from_basis(b); + + tet_volumes[elem_id] = volume; + normalized_flux[elem_id] = + flux(elem_id, 0, 0) / (volume * total_tracklength_l.size()); + normalized_flux_2[elem_id] = + flux(elem_id, 0, 1) / (volume * volume * total_tracklength_l.size()); + + // calculate standard deviation FIXME this is not correct, needs number of + // iterations + sd[elem_id] = + Kokkos::sqrt(normalized_flux_2[elem_id] - + normalized_flux[elem_id] * normalized_flux[elem_id]); + }; + Omega_h::parallel_for(tet_volumes.size(), normalize_flux_with_volume, + "normalize flux"); + + mesh.add_tag(Omega_h::REGION, "volume", 1, Omega_h::Reals(tet_volumes)); + return {normalized_flux}; +} + +void PumiParticleAtElemBoundary::finalizeAndWritePumiFlux( + Omega_h::Mesh &full_mesh, const std::string &filename) { + const auto &normalized_flux = normalizeFlux(full_mesh); + full_mesh.add_tag(Omega_h::REGION, "flux", 1, normalized_flux); + Omega_h::vtk::write_parallel(filename, &full_mesh, 3); +} + +void pumiUpdatePtclPositions(PPPS *ptcls) { + auto x_ps_d = ptcls->get<0>(); + auto xtgt_ps_d = ptcls->get<1>(); + auto updatePtclPos = PS_LAMBDA(const int &, const int &pid, const bool &) { + x_ps_d(pid, 0) = xtgt_ps_d(pid, 0); + x_ps_d(pid, 1) = xtgt_ps_d(pid, 1); + x_ps_d(pid, 2) = xtgt_ps_d(pid, 2); + xtgt_ps_d(pid, 0) = 0.0; + xtgt_ps_d(pid, 1) = 0.0; + xtgt_ps_d(pid, 2) = 0.0; + }; + ps::parallel_for(ptcls, updatePtclPos); +} + +void PumiParticleAtElemBoundary::compute_total_tracklength(PPPS *ptcls) { + auto orig = ptcls->get<0>(); + auto dest = ptcls->get<1>(); + + auto total_tracklength_l = total_tracklength_; + + auto computeTrackLength = + PS_LAMBDA(const int &elemId, const int &pid, const bool &mask) { + Omega_h::Vector<3> p_orig = {orig(pid, 0), orig(pid, 1), orig(pid, 2)}; + Omega_h::Vector<3> p_dest = {dest(pid, 0), dest(pid, 1), dest(pid, 2)}; + Omega_h::Real track_length = Omega_h::norm(p_dest - p_orig); + total_tracklength_l[pid] = track_length; + }; + pumipic::parallel_for(ptcls, computeTrackLength, + "compute total track length"); +} + +// search and update parent elements +//! @param initial initial search finds the initial location of the particles +//! and doesn't tally +void PumiTallyImpl::search_and_rebuild(bool initial, const bool migrate) { + // initial cannot be false when is_pumipic_initialized is false + // may fail if simulated more than one batch + assert((is_pumipic_initialized == false && initial == true) || + (is_pumipic_initialized == true && initial == false)); + p_pumi_particle_at_elem_boundary_handler->mark_initial_as(initial); + auto orig = pumipic_ptcls->get<0>(); + auto dest = pumipic_ptcls->get<1>(); + auto pid = pumipic_ptcls->get<2>(); + + if (p_picparts_->mesh() == nullptr || p_picparts_->mesh()->nelems() == 0) { + fprintf(stderr, "ERROR: Mesh is empty\n"); + } + + // total tracklengths are used to calculate the flux + if (!initial) { + p_pumi_particle_at_elem_boundary_handler->updatePrevXPoint( + pumipic_ptcls.get()); + p_pumi_particle_at_elem_boundary_handler->compute_total_tracklength( + pumipic_ptcls.get()); + } + + bool isFoundAll = p_particle_tracer_->search(migrate); + + if (!isFoundAll) { + printf( + "ERROR: Not all particles are found. May need more loops in search\n"); + } +} + +std::unique_ptr pp_create_particle_structure(Omega_h::Mesh mesh, + pumipic::lid_t numPtcls) { + Omega_h::Int ne = mesh.nelems(); + pumiinopenmc::PPPS::kkLidView ptcls_per_elem("ptcls_per_elem", ne); + pumiinopenmc::PPPS::kkGidView element_gids("element_gids", ne); + + Kokkos::TeamPolicy policy; + + Omega_h::parallel_for( + ne, OMEGA_H_LAMBDA(const Omega_h::LO &i) { element_gids(i) = i; }); + + Omega_h::parallel_for( + mesh.nelems(), OMEGA_H_LAMBDA(Omega_h::LO id) { + ptcls_per_elem[id] = (id == 0) ? numPtcls : 0; + }); #ifdef PUMI_USE_KOKKOS_CUDA - printf("PumiPIC Using GPU for simulation...\n"); - policy = Kokkos::TeamPolicy(10000, 32); + printf("PumiPIC Using GPU for simulation...\n"); + policy = Kokkos::TeamPolicy(10000, 32); #else - printf("PumiPIC Using CPU for simulation...\n"); - policy = Kokkos::TeamPolicy(10000, Kokkos::AUTO()); + printf("PumiPIC Using CPU for simulation...\n"); + policy = + Kokkos::TeamPolicy(10000, Kokkos::AUTO()); #endif - auto ptcls = std::make_unique>(policy, ne, numPtcls, - ptcls_per_elem, element_gids); - - return ptcls; - } - - /* - void set_pumipic_particle_structure_size(int openmc_particles_in_flight, int openmc_work_per_rank, int openmc_n_particles) - { - int64_t n_particles; // TODO have a better way to do it than this - // FIXME why this work_per rank is not set in the settings by now? - - - if (openmc_particles_in_flight == 0 && openmc_work_per_rank == 0) { - printf("While creating PumiPIC particle structure, both max_particles_in_flight and work_per_rank are 0.\n"); - n_particles = (openmc_n_particles != 0) ? openmc_n_particles : pumi_ps_size; - }else if (openmc_particles_in_flight == 0 || openmc_work_per_rank == 0) { - n_particles = std::max(openmc_particles_in_flight, openmc_work_per_rank); - printf("One of max_particles_in_flight or work_per_rank is 0. Setting PumiPIC particle structure size to %d\n", n_particles); - } else { - n_particles = std::min(openmc_particles_in_flight, openmc_work_per_rank); - printf("Setting PumiPIC particle structure size to %d\n", n_particles); - } - - pumi_ps_size = n_particles; - printf("Creteating PumiPIC particle structure with size %d\n", n_particles); - } - */ - - void start_pumi_particles_in_0th_element(Omega_h::Mesh &mesh, pumiinopenmc::PPPS *ptcls) { - // find the centroid of the 0th element - const auto &coords = mesh.coords(); - const auto &tet2node = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; - - Omega_h::Write centroid_of_el0(3, 0.0, "centroid"); - - auto find_centroid_of_el0 = OMEGA_H_LAMBDA(Omega_h::LO - id) { - const auto nodes = Omega_h::gather_verts<4>(tet2node, id); - Omega_h::Few, 4> tet_node_coords = Omega_h::gather_vectors<4, 3>(coords, nodes); - const auto centroid = o::average(tet_node_coords); - centroid_of_el0[0] = centroid[0]; - centroid_of_el0[1] = centroid[1]; - centroid_of_el0[2] = centroid[2]; - }; - Omega_h::parallel_for(1, find_centroid_of_el0, "find centroid of element 0"); - - // assign the location to all particles - auto init_loc = ptcls->get<0>(); - auto pids = ptcls->get<2>(); - auto in_fly = ptcls->get<3>(); - - auto set_initial_positions = PS_LAMBDA(const int &e, const int &pid, const int &mask) { - if (mask > 0) { - pids(pid) = pid; - in_fly(pid) = 1; - init_loc(pid, 0) = centroid_of_el0[0]; - init_loc(pid, 1) = centroid_of_el0[1]; - init_loc(pid, 2) = centroid_of_el0[2]; - } - }; - pumipic::parallel_for(ptcls, set_initial_positions, "set initial particle positions"); - } - - Omega_h::Mesh *PumiTallyImpl::partition_pumipic_mesh() { - Omega_h::Write owners(full_mesh_.nelems(), 0, "owners"); - // all the particles are initialized in element 0 to do an initial search to - // find the starting locations - // of the openmc given particles. - //p_picparts_ = new pumipic::Mesh(full_mesh_, Omega_h::LOs(owners)); - p_picparts_ = std::make_unique(full_mesh_, Omega_h::LOs(owners)); - printf("PumiPIC mesh partitioned\n"); - Omega_h::Mesh *mesh = p_picparts_->mesh(); - return mesh; - } - - void PumiTallyImpl::create_and_initialize_pumi_particle_structure(Omega_h::Mesh *mesh) { - pumipic_ptcls = pp_create_particle_structure(*mesh, pumi_ps_size); - start_pumi_particles_in_0th_element(*mesh, pumipic_ptcls.get()); - p_pumi_particle_at_elem_boundary_handler = - std::make_unique(mesh->nelems(), - pumipic_ptcls->capacity()); - - printf("PumiPIC Mesh and data structure created with %d and %d as particle structure capacity\n", - p_picparts_->mesh()->nelems(), pumipic_ptcls->capacity()); - } - - void PumiTallyImpl::read_pumipic_lib_and_full_mesh(int &argc, char **&argv) { - printf("Reading the Omega_h mesh %s to tally with tracklength estimator\n", oh_mesh_fname.c_str()); - init_pumi_libs(argc, argv); - - if (oh_mesh_fname.empty()) { - printf("[ERROR] Omega_h mesh for PumiPIC is not given. Provide --ohMesh = "); - } - full_mesh_ = Omega_h::binary::read(oh_mesh_fname, &oh_lib); - if (full_mesh_.dim() != 3) { - printf("PumiPIC only works for 3D mesh now.\n"); - } - printf("PumiPIC Loaded mesh %s with %d elements\n", oh_mesh_fname.c_str(), full_mesh_.nelems()); - } - - void PumiTallyImpl::load_pumipic_mesh_and_init_particles(int &argc, char **&argv) { - read_pumipic_lib_and_full_mesh(argc, argv); - Omega_h::Mesh *mesh = partition_pumipic_mesh(); - create_and_initialize_pumi_particle_structure(mesh); - } - - PumiTally::PumiTally(std::string &mesh_filename, int64_t num_particles, int &argc, char **&argv) - : pimpl(std::make_unique(mesh_filename, num_particles, argc, argv)) { - } - - void PumiTally::initialize_particle_location(double *init_particle_positions, int64_t size) { - auto start_time = std::chrono::steady_clock::now(); - - pimpl->initialize_particle_location(init_particle_positions, size); - - std::chrono::duration elapsed_seconds = std::chrono::steady_clock::now() - start_time; - pimpl->tally_times.initialization_time += elapsed_seconds.count(); - } - - void - PumiTally::move_to_next_location(double *particle_destinations, int8_t *flying, double *weights, int64_t size) { - auto start_time = std::chrono::steady_clock::now(); - - pimpl->move_to_next_location(particle_destinations, flying, weights, size); - - std::chrono::duration elapsed_seconds = std::chrono::steady_clock::now() - start_time; - pimpl->tally_times.total_time_to_tally += elapsed_seconds.count(); - } - - void PumiTally::write_pumi_tally_mesh() { - auto start_time = std::chrono::steady_clock::now(); - - pimpl->write_pumi_tally_mesh(); - - std::chrono::duration elapsed_seconds = std::chrono::steady_clock::now() - start_time; - pimpl->tally_times.vtk_file_write_time += elapsed_seconds.count(); - pimpl->tally_times.print_times(); - } + auto ptcls = std::make_unique>( + policy, ne, numPtcls, ptcls_per_elem, element_gids); + + return ptcls; +} + +/* +void set_pumipic_particle_structure_size(int openmc_particles_in_flight, int +openmc_work_per_rank, int openmc_n_particles) +{ + int64_t n_particles; // TODO have a better way to do it than this + // FIXME why this work_per rank is not set in the settings by now? + + + if (openmc_particles_in_flight == 0 && openmc_work_per_rank == 0) { + printf("While creating PumiPIC particle structure, both +max_particles_in_flight and work_per_rank are 0.\n"); n_particles = +(openmc_n_particles != 0) ? openmc_n_particles : pumi_ps_size; }else if +(openmc_particles_in_flight == 0 || openmc_work_per_rank == 0) { n_particles = +std::max(openmc_particles_in_flight, openmc_work_per_rank); printf("One of +max_particles_in_flight or work_per_rank is 0. Setting PumiPIC particle +structure size to %d\n", n_particles); } else { n_particles = +std::min(openmc_particles_in_flight, openmc_work_per_rank); printf("Setting +PumiPIC particle structure size to %d\n", n_particles); + } + + pumi_ps_size = n_particles; + printf("Creteating PumiPIC particle structure with size %d\n", n_particles); +} +*/ + +void start_pumi_particles_in_0th_element(Omega_h::Mesh &mesh, + pumiinopenmc::PPPS *ptcls) { + // find the centroid of the 0th element + const auto &coords = mesh.coords(); + const auto &tet2node = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; + + Omega_h::Write centroid_of_el0(3, 0.0, "centroid"); + + auto find_centroid_of_el0 = OMEGA_H_LAMBDA(Omega_h::LO id) { + const auto nodes = Omega_h::gather_verts<4>(tet2node, id); + Omega_h::Few, 4> tet_node_coords = + Omega_h::gather_vectors<4, 3>(coords, nodes); + const auto centroid = o::average(tet_node_coords); + centroid_of_el0[0] = centroid[0]; + centroid_of_el0[1] = centroid[1]; + centroid_of_el0[2] = centroid[2]; + }; + Omega_h::parallel_for(1, find_centroid_of_el0, "find centroid of element 0"); + + // assign the location to all particles + auto init_loc = ptcls->get<0>(); + auto pids = ptcls->get<2>(); + auto in_fly = ptcls->get<3>(); + + auto set_initial_positions = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if (mask > 0) { + pids(pid) = pid; + in_fly(pid) = 1; + init_loc(pid, 0) = centroid_of_el0[0]; + init_loc(pid, 1) = centroid_of_el0[1]; + init_loc(pid, 2) = centroid_of_el0[2]; + } + }; + pumipic::parallel_for(ptcls, set_initial_positions, + "set initial particle positions"); +} + +Omega_h::Mesh *PumiTallyImpl::partition_pumipic_mesh() { + Omega_h::Write owners(full_mesh_.nelems(), 0, "owners"); + // all the particles are initialized in element 0 to do an initial search to + // find the starting locations + // of the openmc given particles. + // p_picparts_ = new pumipic::Mesh(full_mesh_, Omega_h::LOs(owners)); + p_picparts_ = + std::make_unique(full_mesh_, Omega_h::LOs(owners)); + printf("PumiPIC mesh partitioned\n"); + Omega_h::Mesh *mesh = p_picparts_->mesh(); + return mesh; +} + +void PumiTallyImpl::create_and_initialize_pumi_particle_structure( + Omega_h::Mesh *mesh) { + pumipic_ptcls = pp_create_particle_structure(*mesh, pumi_ps_size); + start_pumi_particles_in_0th_element(*mesh, pumipic_ptcls.get()); + p_pumi_particle_at_elem_boundary_handler = + std::make_unique( + mesh->nelems(), pumipic_ptcls->capacity()); + + printf("PumiPIC Mesh and data structure created with %d and %d as particle " + "structure capacity\n", + p_picparts_->mesh()->nelems(), pumipic_ptcls->capacity()); +} + +void PumiTallyImpl::read_pumipic_lib_and_full_mesh(int &argc, char **&argv) { + printf("Reading the Omega_h mesh %s to tally with tracklength estimator\n", + oh_mesh_fname.c_str()); + init_pumi_libs(argc, argv); + + if (oh_mesh_fname.empty()) { + printf("[ERROR] Omega_h mesh for PumiPIC is not given. Provide --ohMesh = " + ""); + } + full_mesh_ = Omega_h::binary::read(oh_mesh_fname, &oh_lib); + if (full_mesh_.dim() != 3) { + printf("PumiPIC only works for 3D mesh now.\n"); + } + printf("PumiPIC Loaded mesh %s with %d elements\n", oh_mesh_fname.c_str(), + full_mesh_.nelems()); +} + +void PumiTallyImpl::load_pumipic_mesh_and_init_particles(int &argc, + char **&argv) { + read_pumipic_lib_and_full_mesh(argc, argv); + Omega_h::Mesh *mesh = partition_pumipic_mesh(); + create_and_initialize_pumi_particle_structure(mesh); +} + +PumiTally::PumiTally(std::string &mesh_filename, int64_t num_particles, + int &argc, char **&argv) + : pimpl(std::make_unique(mesh_filename, num_particles, argc, + argv)) {} + +void PumiTally::initialize_particle_location(double *init_particle_positions, + int64_t size) { + auto start_time = std::chrono::steady_clock::now(); + + pimpl->initialize_particle_location(init_particle_positions, size); + + std::chrono::duration elapsed_seconds = + std::chrono::steady_clock::now() - start_time; + pimpl->tally_times.initialization_time += elapsed_seconds.count(); +} + +void PumiTally::move_to_next_location(double *particle_destinations, + int8_t *flying, double *weights, + int64_t size) { + auto start_time = std::chrono::steady_clock::now(); + + pimpl->move_to_next_location(particle_destinations, flying, weights, size); + + std::chrono::duration elapsed_seconds = + std::chrono::steady_clock::now() - start_time; + pimpl->tally_times.total_time_to_tally += elapsed_seconds.count(); +} + +void PumiTally::write_pumi_tally_mesh() { + auto start_time = std::chrono::steady_clock::now(); + + pimpl->write_pumi_tally_mesh(); + + std::chrono::duration elapsed_seconds = + std::chrono::steady_clock::now() - start_time; + pimpl->tally_times.vtk_file_write_time += elapsed_seconds.count(); + pimpl->tally_times.print_times(); +} } // namespace pumiinopenmc diff --git a/test/test_pumi_tally_impl_methods.cpp b/test/test_pumi_tally_impl_methods.cpp index 6319a68..0dd130b 100644 --- a/test/test_pumi_tally_impl_methods.cpp +++ b/test/test_pumi_tally_impl_methods.cpp @@ -1,419 +1,475 @@ // // Created by Fuad Hasan on 2/3/25. // -#include #include #include #include +#include -// ********************************** Notes ************************************************************// -// - Sections are not working for some reason. Saying using MPI functions before or after MPI init or finalize +// ********************************** Notes +// ************************************************************// +// - Sections are not working for some reason. Saying using MPI functions before +// or after MPI init or finalize // - sections are marked with comments for now // - TODO Remove including cpp by creating another internal header -// - Look at this gist to verify this in python: https://gist.github.com/Fuad-HH/5e0aed99f271617e283e9108091fb1cb +// - Look at this gist to verify this in python: +// https://gist.github.com/Fuad-HH/5e0aed99f271617e283e9108091fb1cb // *****************************************************************************************************// // TODO: Remove it by having another header file #include "pumipic_particle_data_structure.cpp" bool is_close(const double a, const double b, double tol = 1e-8) { - return std::abs(a - b) < tol; + return std::abs(a - b) < tol; } -OMEGA_H_INLINE bool is_close_d(const double a, const double b, double tol = 1e-8) { - return Kokkos::abs(a - b) < tol; +OMEGA_H_INLINE bool is_close_d(const double a, const double b, + double tol = 1e-8) { + return Kokkos::abs(a - b) < tol; } TEST_CASE("Test Impl Class Functions") { - auto lib = Omega_h::Library{}; - auto world = lib.world(); - // simplest 3D mesh - auto mesh = - Omega_h::build_box(world, OMEGA_H_SIMPLEX, 1, 1, 1, 1, 1, 1, false); - printf("[INFO] Mesh created with %d vertices and %d faces\n", - mesh.nverts(), mesh.nfaces()); - - // create particle structure with 5 particles - int num_ptcls = 5; - - // TODO remove this read_write - int argc = 0; - char **argv; - std::string temp_file_name = "mesh.osh"; - Omega_h::binary::write(temp_file_name, &mesh); - fprintf(stdout, "[INFO] Mesh written to file :%s\n", temp_file_name.c_str()); - - //****************************** Checks Regarding Constructor ******************************************// - //*******************************************************************************************************// - - // create particle structure with 5 particles - std::unique_ptr p_pumi_tallyimpl = std::make_unique( - temp_file_name, num_ptcls, - argc, argv); - fprintf(stdout, "[INFO] Particle structure created successfully\n"); - - {// * Check other array sizes - auto device_pos_l = p_pumi_tallyimpl->device_pos_buffer_; - auto device_adv_l = p_pumi_tallyimpl->device_in_adv_que_; - auto device_wgt_l = p_pumi_tallyimpl->weights_; - - REQUIRE(device_pos_l.size() == num_ptcls * 3); - REQUIRE(device_adv_l.size() == num_ptcls); - REQUIRE(device_wgt_l.size() == num_ptcls); - - // * Check full mesh - REQUIRE(p_pumi_tallyimpl->full_mesh_.nelems() == 6); - REQUIRE(p_pumi_tallyimpl->full_mesh_.dim() == 3); - - // * Check the picparts - REQUIRE(p_pumi_tallyimpl->p_picparts_->isFullMesh() == true); - - // * Check created particle structure - REQUIRE(p_pumi_tallyimpl->pumipic_ptcls->nPtcls() == num_ptcls); - REQUIRE(p_pumi_tallyimpl->pumipic_ptcls->capacity() >= num_ptcls); - REQUIRE(p_pumi_tallyimpl->pumipic_ptcls->nElems() == mesh.nelems()); + auto lib = Omega_h::Library{}; + auto world = lib.world(); + // simplest 3D mesh + auto mesh = + Omega_h::build_box(world, OMEGA_H_SIMPLEX, 1, 1, 1, 1, 1, 1, false); + printf("[INFO] Mesh created with %d vertices and %d faces\n", mesh.nverts(), + mesh.nfaces()); + + // create particle structure with 5 particles + int num_ptcls = 5; + + // TODO remove this read_write + int argc = 0; + char **argv; + std::string temp_file_name = "mesh.osh"; + Omega_h::binary::write(temp_file_name, &mesh); + fprintf(stdout, "[INFO] Mesh written to file :%s\n", temp_file_name.c_str()); + + //****************************** Checks Regarding Constructor + //******************************************// + //*******************************************************************************************************// + + // create particle structure with 5 particles + std::unique_ptr p_pumi_tallyimpl = + std::make_unique(temp_file_name, num_ptcls, + argc, argv); + fprintf(stdout, "[INFO] Particle structure created successfully\n"); + + { // * Check other array sizes + auto device_pos_l = p_pumi_tallyimpl->device_pos_buffer_; + auto device_adv_l = p_pumi_tallyimpl->device_in_adv_que_; + auto device_wgt_l = p_pumi_tallyimpl->weights_; + + REQUIRE(device_pos_l.size() == num_ptcls * 3); + REQUIRE(device_adv_l.size() == num_ptcls); + REQUIRE(device_wgt_l.size() == num_ptcls); + + // * Check full mesh + REQUIRE(p_pumi_tallyimpl->full_mesh_.nelems() == 6); + REQUIRE(p_pumi_tallyimpl->full_mesh_.dim() == 3); + + // * Check the picparts + REQUIRE(p_pumi_tallyimpl->p_picparts_->isFullMesh() == true); + + // * Check created particle structure + REQUIRE(p_pumi_tallyimpl->pumipic_ptcls->nPtcls() == num_ptcls); + REQUIRE(p_pumi_tallyimpl->pumipic_ptcls->capacity() >= num_ptcls); + REQUIRE(p_pumi_tallyimpl->pumipic_ptcls->nElems() == mesh.nelems()); + } + + { // * Check if particles have origin at 0th element + auto origin = p_pumi_tallyimpl->pumipic_ptcls->get<0>(); + Omega_h::Vector<3> cell_centroid_of_elem0{0.5, 0.75, 0.25}; + Omega_h::Write cell_centroids( + p_pumi_tallyimpl->pumipic_ptcls->nPtcls() * 3, 0.0, "check_sum"); + auto check_init_at_0th_elem = + PS_LAMBDA(const auto &el, const auto &pid, const auto &mask) { + if (mask > 0) { + cell_centroids[pid * 3] = origin(pid, 0); + cell_centroids[pid * 3 + 1] = origin(pid, 1); + cell_centroids[pid * 3 + 2] = origin(pid, 2); + } + }; + pumipic::parallel_for(p_pumi_tallyimpl->pumipic_ptcls.get(), + check_init_at_0th_elem, + "check if particles are intialized at 0th element"); + Omega_h::HostWrite cell_centroids_host(cell_centroids); + for (int pid = 0; pid < num_ptcls; ++pid) { + printf("Particle %d position: (%.16f, %.16f, %.16f)\n", pid, + cell_centroids_host[pid * 3], cell_centroids_host[pid * 3 + 1], + cell_centroids_host[pid * 3 + 2]); + REQUIRE( + is_close(cell_centroids_host[pid * 3], cell_centroid_of_elem0[0])); + REQUIRE(is_close(cell_centroids_host[pid * 3 + 1], + cell_centroid_of_elem0[1])); + REQUIRE(is_close(cell_centroids_host[pid * 3 + 2], + cell_centroid_of_elem0[2])); } + } - {// * Check if particles have origin at 0th element - auto origin = p_pumi_tallyimpl->pumipic_ptcls->get<0>(); - Omega_h::Vector<3> cell_centroid_of_elem0{0.5, 0.75, 0.25}; - Omega_h::Write cell_centroids(p_pumi_tallyimpl->pumipic_ptcls->nPtcls() * 3, 0.0, "check_sum"); - auto check_init_at_0th_elem = PS_LAMBDA(const auto &el, const auto &pid, const auto &mask) { - if (mask > 0) { - cell_centroids[pid * 3] = origin(pid, 0); - cell_centroids[pid * 3 + 1] = origin(pid, 1); - cell_centroids[pid * 3 + 2] = origin(pid, 2); - } - }; - pumipic::parallel_for(p_pumi_tallyimpl->pumipic_ptcls.get(), check_init_at_0th_elem, - "check if particles are intialized at 0th element"); - Omega_h::HostWrite cell_centroids_host(cell_centroids); - for (int pid = 0; pid < num_ptcls; ++pid) { - printf("Particle %d position: (%.16f, %.16f, %.16f)\n", pid, - cell_centroids_host[pid * 3], - cell_centroids_host[pid * 3 + 1], - cell_centroids_host[pid * 3 + 2]); - REQUIRE(is_close(cell_centroids_host[pid * 3], cell_centroid_of_elem0[0])); - REQUIRE(is_close(cell_centroids_host[pid * 3 + 1], cell_centroid_of_elem0[1])); - REQUIRE(is_close(cell_centroids_host[pid * 3 + 2], cell_centroid_of_elem0[2])); - } - } + //************************************ Checks Regarding Initializing Particle + //Locations ***********************// + //*************************************************************************************************************// - //************************************ Checks Regarding Initializing Particle Locations ***********************// - //*************************************************************************************************************// - - {// * Note: all 5 particles will start their journey as follows: - // ray_origin = [0.1, 0.4, 0.5] in cell 2 - // ray_end = [1.1, 0.4, 0.5] passing through cells 2, 3, 4 and finally leaving the box - std::vector init_particle_positions(num_ptcls * 3); - for (int pid = 0; pid < num_ptcls; ++pid) { - init_particle_positions[pid * 3] = 0.1; - init_particle_positions[pid * 3 + 1] = 0.4; - init_particle_positions[pid * 3 + 2] = 0.5; - } - p_pumi_tallyimpl->initialize_particle_location(init_particle_positions.data(), init_particle_positions.size()); - - {// * Check if particle positions are copied properly in the device - // ? is it okay to check with OMEGA_H_CHECK - auto device_pos_buffer_l = p_pumi_tallyimpl->device_pos_buffer_; - auto check_device_init_pos = OMEGA_H_LAMBDA(int pid) { - OMEGA_H_CHECK_PRINTF(is_close_d(device_pos_buffer_l[pid * 3], 0.1), - "Particle position copy to device error 0: %.16f %.16f\n", - device_pos_buffer_l[pid * 3], 0.1); - OMEGA_H_CHECK_PRINTF(is_close_d(device_pos_buffer_l[pid * 3 + 1], 0.4), - "Particle position copy to device error 0: %.16f %.16f\n", - device_pos_buffer_l[pid * 3 + 1], 0.4); - OMEGA_H_CHECK_PRINTF(is_close_d(device_pos_buffer_l[pid * 3 + 2], 0.5), - "Particle position copy to device error 0: %.16f %.16f\n", - device_pos_buffer_l[pid * 3 + 2], 0.5); - }; - Omega_h::parallel_for(num_ptcls, check_device_init_pos, - "Check if the init particle pos are copied to device correctly"); - } + { // * Note: all 5 particles will start their journey as follows: + // ray_origin = [0.1, 0.4, 0.5] in cell 2 + // ray_end = [1.1, 0.4, 0.5] passing through cells 2, 3, 4 and + // finally leaving the box + std::vector init_particle_positions(num_ptcls * 3); + for (int pid = 0; pid < num_ptcls; ++pid) { + init_particle_positions[pid * 3] = 0.1; + init_particle_positions[pid * 3 + 1] = 0.4; + init_particle_positions[pid * 3 + 2] = 0.5; } - - {// * Check if all particles reached element 2 - auto elem_ids_host = Omega_h::HostRead(p_pumi_tallyimpl->p_particle_tracer_->getElementIds()); - REQUIRE(elem_ids_host.size() == p_pumi_tallyimpl->pumipic_ptcls->capacity()); - for (int pid = 0; pid < num_ptcls; ++pid) { - REQUIRE(elem_ids_host[pid] == 2); - } - - // * The fluxes should be zero since the init doesn't calculate flux - auto flux_l = p_pumi_tallyimpl->p_pumi_particle_at_elem_boundary_handler->flux_; - Omega_h::HostWrite flux_host(flux_l); - REQUIRE(flux_host.size() == mesh.nelems()); - for (int el = 0; el < flux_host.size(); ++el) { - printf("Flux after init run %d[%.16f]\n", el, flux_host[el]); - REQUIRE(is_close(flux_host[el], 0.0)); - } + p_pumi_tallyimpl->initialize_particle_location( + init_particle_positions.data(), init_particle_positions.size()); + + { // * Check if particle positions are copied properly in the device + // ? is it okay to check with OMEGA_H_CHECK + auto device_pos_buffer_l = p_pumi_tallyimpl->device_pos_buffer_; + auto check_device_init_pos = OMEGA_H_LAMBDA(int pid) { + OMEGA_H_CHECK_PRINTF( + is_close_d(device_pos_buffer_l[pid * 3], 0.1), + "Particle position copy to device error 0: %.16f %.16f\n", + device_pos_buffer_l[pid * 3], 0.1); + OMEGA_H_CHECK_PRINTF( + is_close_d(device_pos_buffer_l[pid * 3 + 1], 0.4), + "Particle position copy to device error 0: %.16f %.16f\n", + device_pos_buffer_l[pid * 3 + 1], 0.4); + OMEGA_H_CHECK_PRINTF( + is_close_d(device_pos_buffer_l[pid * 3 + 2], 0.5), + "Particle position copy to device error 0: %.16f %.16f\n", + device_pos_buffer_l[pid * 3 + 2], 0.5); + }; + Omega_h::parallel_for( + num_ptcls, check_device_init_pos, + "Check if the init particle pos are copied to device correctly"); } + } - //******************************* Checks Move to Next Location *************************************************// - //**************************************************************************************************************// - - // All the particles will now go to ray end - std::vector particle_destination(num_ptcls * 3); - std::vector weights(num_ptcls, 1.0); // same weights - + { // * Check if all particles reached element 2 + auto elem_ids_host = Omega_h::HostRead( + p_pumi_tallyimpl->p_particle_tracer_->getElementIds()); + REQUIRE(elem_ids_host.size() == + p_pumi_tallyimpl->pumipic_ptcls->capacity()); for (int pid = 0; pid < num_ptcls; ++pid) { - particle_destination[pid * 3] = 1.2; - particle_destination[pid * 3 + 1] = 0.4; - particle_destination[pid * 3 + 2] = 0.5; + REQUIRE(elem_ids_host[pid] == 2); } + // * The fluxes should be zero since the init doesn't calculate flux + auto flux_l = + p_pumi_tallyimpl->p_pumi_particle_at_elem_boundary_handler->flux_; + auto flux_host = Kokkos::create_mirror_view(flux_l); + Kokkos::deep_copy(flux_host, flux_l); + REQUIRE(flux_host.extent(0) == mesh.nelems()); + for (int el = 0; el < flux_host.size(); ++el) { + printf("Flux after init run %d[%.16f]\n", el, flux_host(el, 0, 0)); + REQUIRE(is_close(flux_host(el, 0, 0), 0.0)); + } + } + + //******************************* Checks Move to Next Location + //*************************************************// + //**************************************************************************************************************// + + // All the particles will now go to ray end + std::vector particle_destination(num_ptcls * 3); + std::vector weights(num_ptcls, 1.0); // same weights + + for (int pid = 0; pid < num_ptcls; ++pid) { + particle_destination[pid * 3] = 1.2; + particle_destination[pid * 3 + 1] = 0.4; + particle_destination[pid * 3 + 2] = 0.5; + } + + { // * Check copy data to device and reset flying + std::vector flying(num_ptcls, 1); // all are flying now + + REQUIRE(particle_destination.size() == 3 * p_pumi_tallyimpl->pumi_ps_size); + p_pumi_tallyimpl->copy_data_to_device(particle_destination.data()); + p_pumi_tallyimpl->copy_and_reset_flying_flag(flying.data()); + p_pumi_tallyimpl->copy_weights(weights.data()); + + auto particle_destinations_l = p_pumi_tallyimpl->device_pos_buffer_; + Omega_h::HostWrite particle_destination_l_host( + particle_destinations_l); + auto particle_weight_l = p_pumi_tallyimpl->weights_; + Omega_h::HostWrite particle_weight_l_host(particle_weight_l); + auto particle_flying_l = p_pumi_tallyimpl->device_in_adv_que_; + Omega_h::HostWrite particle_flying_l_host(particle_flying_l); - { // * Check copy data to device and reset flying - std::vector flying(num_ptcls, 1); // all are flying now - - REQUIRE(particle_destination.size() == 3 * p_pumi_tallyimpl->pumi_ps_size); - p_pumi_tallyimpl->copy_data_to_device(particle_destination.data()); - p_pumi_tallyimpl->copy_and_reset_flying_flag(flying.data()); - p_pumi_tallyimpl->copy_weights(weights.data()); - - auto particle_destinations_l = p_pumi_tallyimpl->device_pos_buffer_; - Omega_h::HostWrite particle_destination_l_host(particle_destinations_l); - auto particle_weight_l = p_pumi_tallyimpl->weights_; - Omega_h::HostWrite particle_weight_l_host(particle_weight_l); - auto particle_flying_l = p_pumi_tallyimpl->device_in_adv_que_; - Omega_h::HostWrite particle_flying_l_host(particle_flying_l); - - for (int pid = 0; pid < num_ptcls; ++pid) { - REQUIRE(is_close(particle_destination_l_host[pid * 3], 1.2)); - REQUIRE(is_close(particle_destination_l_host[pid * 3 + 1], 0.4)); - REQUIRE(is_close(particle_destination_l_host[pid * 3 + 2], 0.5)); + for (int pid = 0; pid < num_ptcls; ++pid) { + REQUIRE(is_close(particle_destination_l_host[pid * 3], 1.2)); + REQUIRE(is_close(particle_destination_l_host[pid * 3 + 1], 0.4)); + REQUIRE(is_close(particle_destination_l_host[pid * 3 + 2], 0.5)); - REQUIRE(is_close(particle_weight_l_host[pid], 1.0)); - REQUIRE(particle_flying_l_host[pid] == 1); + REQUIRE(is_close(particle_weight_l_host[pid], 1.0)); + REQUIRE(particle_flying_l_host[pid] == 1); - REQUIRE(flying[pid] == 0); - } + REQUIRE(flying[pid] == 0); } - - { // not a check, just move - std::vector flying(num_ptcls, 1); // reset them again to 1 - p_pumi_tallyimpl->move_to_next_location(particle_destination.data(), flying.data(), weights.data(), - particle_destination.size()); + } + + { // not a check, just move + std::vector flying(num_ptcls, 1); // reset them again to 1 + p_pumi_tallyimpl->move_to_next_location(particle_destination.data(), + flying.data(), weights.data(), + particle_destination.size()); + } + + { // * Check if the particles correctly reaches element 4 + auto elem_ids_local = p_pumi_tallyimpl->p_particle_tracer_->getElementIds(); + Omega_h::HostRead elem_ids_local_host(elem_ids_local); + for (int pid = 0; pid < num_ptcls; ++pid) { + printf("[INFO] Particles reached elem %d\n", elem_ids_local_host[pid]); + REQUIRE(elem_ids_local_host[pid] == 4); } - - {// * Check if the particles correctly reaches element 4 - auto elem_ids_local = p_pumi_tallyimpl->p_particle_tracer_->getElementIds(); - Omega_h::HostRead elem_ids_local_host(elem_ids_local); - for (int pid = 0; pid < num_ptcls; ++pid) { - printf("[INFO] Particles reached elem %d\n", elem_ids_local_host[pid]); - REQUIRE(elem_ids_local_host[pid] == 4); - } + } + + { // * Check if particles reached destinations properly, weights and flying + // flags are copied properly + auto new_origin = p_pumi_tallyimpl->pumipic_ptcls->get<0>(); + auto particle_flyign = p_pumi_tallyimpl->pumipic_ptcls->get<3>(); + auto particle_weights = p_pumi_tallyimpl->pumipic_ptcls->get<4>(); + auto check_copied_properties = + PS_LAMBDA(const auto &e, const auto &pid, const auto &mask) { + if (mask > 0) { + printf("Particle new origin (%f, %f, %f), flying %d, weight %f\n", + new_origin(pid, 0), new_origin(pid, 1), new_origin(pid, 2), + particle_flyign(pid), particle_weights(pid)); + // fixme The new position should be 1.0 rather than 1.1 since it goes + // out + OMEGA_H_CHECK_PRINTF(is_close_d(new_origin(pid, 0), 1.0), + "Particle destination not copied properly %.16f\n", + new_origin(pid, 0)); + OMEGA_H_CHECK_PRINTF(is_close_d(new_origin(pid, 1), 0.4), + "Particle destination not copied properly %.16f\n", + new_origin(pid, 1)); + OMEGA_H_CHECK_PRINTF(is_close_d(new_origin(pid, 2), 0.5), + "Particle destination not copied properly %.16f\n", + new_origin(pid, 2)); + + OMEGA_H_CHECK_PRINTF(particle_flyign(pid) == 1, + "Particle flying not copied correctly, found %d\n", + particle_flyign(pid)); + OMEGA_H_CHECK_PRINTF( + is_close_d(particle_weights(pid), 1.0), + "Particle weight not copied properly, found %.16f\n", + particle_weights(pid)); + } + }; + pumipic::parallel_for(p_pumi_tallyimpl->pumipic_ptcls.get(), + check_copied_properties, + "check if data copied before move"); + } + + { // * Check flux + // * Note: The particles are going through elems 2, 3, 4. The lengths are: + // 0.3, 0.1, and 0.5 (times 5 for 5 particles) + auto flux_local = + p_pumi_tallyimpl->p_pumi_particle_at_elem_boundary_handler->flux_; + auto flux_host = Kokkos::create_mirror_view(flux_local); + Kokkos::deep_copy(flux_host, flux_local); + printf("The fluxes are %d[%f] %d[%f] %d[%f] %d[%f] %d[%f] %d[%f]\n", 0, + flux_host(0, 0, 0), 1, flux_host(1, 0, 0), 2, flux_host(2, 0, 0), 3, + flux_host(3, 0, 0), 4, flux_host(4, 0, 0), 5, flux_host(5, 0, 0)); + REQUIRE(is_close(flux_host(0, 0, 0), 0.0)); + REQUIRE(is_close(flux_host(1, 0, 0), 0.0)); + REQUIRE(is_close(flux_host(2, 0, 0), 0.3 * num_ptcls)); + REQUIRE(is_close(flux_host(3, 0, 0), 0.1 * num_ptcls)); + REQUIRE(is_close(flux_host(4, 0, 0), 0.5 * num_ptcls)); + REQUIRE(is_close(flux_host(5, 0, 0), 0.0)); + } + + { // * Check if flux is accumulated properly if particles *move again* + // ********************************************** setup + // ******************************************************// Note: particle 0 + // and 2 will move to two different locations starting from (1.0, 0.4, 0.5) + // in element 4 to (0.15, 0.05, 0.2) and (0.85, 0.05, 0.1) in 3 and 4 + // respectively Particle 0 intersects at [0.22727273 0.08181818 0.22727273] + // and 3 doesn't go out of 4 now the weights are 2 and 0.5 respectively + Omega_h::HostWrite next_positions(3 * num_ptcls); + std::vector flying_flags(num_ptcls, 0); + std::vector particle_weights(num_ptcls, 1); + for (int pid = 0; pid < num_ptcls; ++pid) { + if (pid == 0) { + next_positions[3 * pid] = 0.15; + next_positions[3 * pid + 1] = 0.05; + next_positions[3 * pid + 2] = 0.20; + + flying_flags[pid] = 1; + particle_weights[pid] = 2.0; + } else if (pid == 2) { + next_positions[3 * pid] = 0.85; + next_positions[3 * pid + 1] = 0.05; + next_positions[3 * pid + 2] = 0.10; + + flying_flags[pid] = 1; + particle_weights[pid] = 0.5; + } else { + next_positions[3 * pid] = 1.0; + next_positions[3 * pid + 1] = 0.4; + next_positions[3 * pid + 2] = 0.5; + + flying_flags[pid] = 0; + particle_weights[pid] = 1; + } } - - {// * Check if particles reached destinations properly, weights and flying flags are copied properly - auto new_origin = p_pumi_tallyimpl->pumipic_ptcls->get<0>(); - auto particle_flyign = p_pumi_tallyimpl->pumipic_ptcls->get<3>(); - auto particle_weights = p_pumi_tallyimpl->pumipic_ptcls->get<4>(); - auto check_copied_properties = PS_LAMBDA(const auto &e, const auto &pid, const auto &mask) { - if (mask > 0) { - printf("Particle new origin (%f, %f, %f), flying %d, weight %f\n", new_origin(pid, 0), - new_origin(pid, 1), new_origin(pid, 2), - particle_flyign(pid), particle_weights(pid)); - // fixme The new position should be 1.0 rather than 1.1 since it goes out - OMEGA_H_CHECK_PRINTF(is_close_d(new_origin(pid, 0), 1.0), - "Particle destination not copied properly %.16f\n", new_origin(pid, 0)); - OMEGA_H_CHECK_PRINTF(is_close_d(new_origin(pid, 1), 0.4), - "Particle destination not copied properly %.16f\n", new_origin(pid, 1)); - OMEGA_H_CHECK_PRINTF(is_close_d(new_origin(pid, 2), 0.5), - "Particle destination not copied properly %.16f\n", new_origin(pid, 2)); - - OMEGA_H_CHECK_PRINTF(particle_flyign(pid) == 1, "Particle flying not copied correctly, found %d\n", - particle_flyign(pid)); - OMEGA_H_CHECK_PRINTF(is_close_d(particle_weights(pid), 1.0), - "Particle weight not copied properly, found %.16f\n", particle_weights(pid)); - } - }; - pumipic::parallel_for(p_pumi_tallyimpl->pumipic_ptcls.get(), check_copied_properties, - "check if data copied before move"); + p_pumi_tallyimpl->move_to_next_location( + next_positions.data(), flying_flags.data(), particle_weights.data(), + next_positions.size()); + // ***********************************************************************************************************// + + { // * check new origins + Omega_h::Write new_positions_device(next_positions); + auto new_origin = p_pumi_tallyimpl->pumipic_ptcls->get<0>(); + auto check_new_origin = PS_LAMBDA(int e, int pid, int mask) { + if (mask > 0) { + printf("New positions after 2nd Move pid %d(expected|found): %f|%f, " + "%f|%f, %f|%f\n", + pid, new_positions_device[pid * 3], new_origin(pid, 0), + new_positions_device[pid * 3 + 1], new_origin(pid, 1), + new_positions_device[pid * 3 + 2], new_origin(pid, 2)); + for (int i = 0; i < 3; i++) { + OMEGA_H_CHECK_PRINTF( + is_close_d(new_origin(pid, i), + new_positions_device[pid * 3 + i]), + "Origin didn't update properly %d %d: %f ~= %f\n", pid, i, + new_origin(pid, i), new_positions_device[pid * 3 + 1]); + } + } + }; + pumipic::parallel_for(p_pumi_tallyimpl->pumipic_ptcls.get(), + check_new_origin, + "check new origins after 2nd move"); + printf("2nd move transported successfully!\n"); } - {// * Check flux - // * Note: The particles are going through elems 2, 3, 4. The lengths are: - // 0.3, 0.1, and 0.5 (times 5 for 5 particles) - auto flux_local = p_pumi_tallyimpl->p_pumi_particle_at_elem_boundary_handler->flux_; - Omega_h::HostWrite flux_host(flux_local); - printf("The fluxes are %d[%f] %d[%f] %d[%f] %d[%f] %d[%f] %d[%f]\n", 0, flux_host[0], 1, flux_host[1], 2, - flux_host[2], 3, flux_host[3], 4, flux_host[4], 5, flux_host[5]); - REQUIRE(is_close(flux_host[0], 0.0)); - REQUIRE(is_close(flux_host[1], 0.0)); - REQUIRE(is_close(flux_host[2], 0.3 * num_ptcls)); - REQUIRE(is_close(flux_host[3], 0.1 * num_ptcls)); - REQUIRE(is_close(flux_host[4], 0.5 * num_ptcls)); - REQUIRE(is_close(flux_host[5], 0.0)); + { // check destination element + auto elem_id_local = + p_pumi_tallyimpl->p_particle_tracer_->getElementIds(); + Omega_h::HostRead elem_id_host(elem_id_local); + printf("After the 2nd move, the current ids are: %d, %d, %d, %d, %d\n", + elem_id_host[0], elem_id_host[1], elem_id_host[2], elem_id_host[3], + elem_id_host[4]); + REQUIRE(elem_id_host[0] == 3); // moves to 3 + REQUIRE(elem_id_host[1] == 4); + REQUIRE(elem_id_host[2] == 4); // remains inside 4 + REQUIRE(elem_id_host[3] == 4); + REQUIRE(elem_id_host[4] == 4); } - {// * Check if flux is accumulated properly if particles *move again* - // ********************************************** setup ******************************************************// - // Note: particle 0 and 2 will move to two different locations - // starting from (1.0, 0.4, 0.5) in element 4 to (0.15, 0.05, 0.2) and (0.85, 0.05, 0.1) - // in 3 and 4 respectively - // Particle 0 intersects at [0.22727273 0.08181818 0.22727273] and 3 doesn't go out of 4 - // now the weights are 2 and 0.5 respectively - Omega_h::HostWrite next_positions(3 * num_ptcls); - std::vector flying_flags(num_ptcls, 0); - std::vector particle_weights(num_ptcls, 1); - for (int pid = 0; pid < num_ptcls; ++pid) { - if (pid == 0) { - next_positions[3 * pid] = 0.15; - next_positions[3 * pid + 1] = 0.05; - next_positions[3 * pid + 2] = 0.20; - - flying_flags[pid] = 1; - particle_weights[pid] = 2.0; - } else if (pid == 2) { - next_positions[3 * pid] = 0.85; - next_positions[3 * pid + 1] = 0.05; - next_positions[3 * pid + 2] = 0.10; - - flying_flags[pid] = 1; - particle_weights[pid] = 0.5; - } else { - next_positions[3 * pid] = 1.0; - next_positions[3 * pid + 1] = 0.4; - next_positions[3 * pid + 2] = 0.5; - - flying_flags[pid] = 0; - particle_weights[pid] = 1; - } - } - p_pumi_tallyimpl->move_to_next_location(next_positions.data(), flying_flags.data(), particle_weights.data(), - next_positions.size()); - // ***********************************************************************************************************// - - { // * check new origins - Omega_h::Write new_positions_device(next_positions); - auto new_origin = p_pumi_tallyimpl->pumipic_ptcls->get<0>(); - auto check_new_origin = PS_LAMBDA(int e, int pid, int mask) { - if (mask > 0) { - printf("New positions after 2nd Move pid %d(expected|found): %f|%f, %f|%f, %f|%f\n", pid, - new_positions_device[pid * 3], new_origin(pid, 0), - new_positions_device[pid * 3 + 1], new_origin(pid, 1), - new_positions_device[pid * 3 + 2], new_origin(pid, 2)); - for (int i = 0; i < 3; i++) { - OMEGA_H_CHECK_PRINTF(is_close_d(new_origin(pid, i), new_positions_device[pid * 3 + i]), - "Origin didn't update properly %d %d: %f ~= %f\n", - pid, i, new_origin(pid, i), new_positions_device[pid * 3 + 1]); - } - } - }; - pumipic::parallel_for(p_pumi_tallyimpl->pumipic_ptcls.get(), check_new_origin, - "check new origins after 2nd move"); - printf("2nd move transported successfully!\n"); - } - - {// check destination element - auto elem_id_local = p_pumi_tallyimpl->p_particle_tracer_->getElementIds(); - Omega_h::HostRead elem_id_host(elem_id_local); - printf("After the 2nd move, the current ids are: %d, %d, %d, %d, %d\n", - elem_id_host[0], elem_id_host[1], elem_id_host[2], elem_id_host[3], - elem_id_host[4]); - REQUIRE(elem_id_host[0] == 3); // moves to 3 - REQUIRE(elem_id_host[1] == 4); - REQUIRE(elem_id_host[2] == 4); // remains inside 4 - REQUIRE(elem_id_host[3] == 4); - REQUIRE(elem_id_host[4] == 4); - } - - {//* Check flux - //* note: - //* particle 3s contributions will go to element 4 and 3 - //* particle 5's contribution will go to element 4 only - //* segment lengths of 3 are: 0.8790232192610158 and 0.08793076822136835 in 4 and 3 respectively - //* segment length of 5 is:b 0.552268050859363 in 4 - auto flux_local = p_pumi_tallyimpl->p_pumi_particle_at_elem_boundary_handler->flux_; - Omega_h::HostWrite flux_host(flux_local); - Omega_h::HostWrite flux_expected(flux_local); - flux_expected[3] = 0.1 * num_ptcls + 0.08790490988459178 * 2.0; - flux_expected[4] = 0.5 * num_ptcls + 0.879049070406094 * 2.0 + 0.552268050859363 * 0.5; - - printf("The fluxes after 2nd move \nelem_id[found, expected] \n%d[%f,%f] \n%d[%f,%f] \n%d[%f,%f] \n%d[%f,%f] \n%d[%f,%f] \n%d[%f,%f]\n", - 0, flux_host[0], flux_expected[0], 1, flux_host[1], flux_expected[1], - 2, flux_host[2], flux_expected[2], 3, flux_host[3], flux_expected[3], - 4, flux_host[4], flux_expected[4], 5, flux_host[5], flux_expected[5]); - REQUIRE(is_close(flux_host[0], flux_expected[0])); - REQUIRE(is_close(flux_host[1], flux_expected[1])); - REQUIRE(is_close(flux_host[2], flux_expected[2])); - REQUIRE(is_close(flux_host[3], flux_expected[3])); - REQUIRE(is_close(flux_host[4], flux_expected[4])); - REQUIRE(is_close(flux_host[5], flux_expected[5])); - } + { //* Check flux + //* note: + //* particle 3s contributions will go to element 4 and 3 + //* particle 5's contribution will go to element 4 only + //* segment lengths of 3 are: 0.8790232192610158 and 0.08793076822136835 + //in 4 and 3 respectively + //* segment length of 5 is:b 0.552268050859363 in 4 + auto flux_local = + p_pumi_tallyimpl->p_pumi_particle_at_elem_boundary_handler->flux_; + auto flux_host = Kokkos::create_mirror_view(flux_local); + Kokkos::deep_copy(flux_host, flux_local); + auto flux_expected = Kokkos::create_mirror_view(flux_local); + Kokkos::deep_copy(flux_expected, flux_local); + flux_expected(3, 0, 0) = 0.1 * num_ptcls + 0.08790490988459178 * 2.0; + flux_expected(4, 0, 0) = + 0.5 * num_ptcls + 0.879049070406094 * 2.0 + 0.552268050859363 * 0.5; + + printf("The fluxes after 2nd move \nelem_id[found, expected] \n%d[%f,%f] " + "\n%d[%f,%f] \n%d[%f,%f] \n%d[%f,%f] \n%d[%f,%f] \n%d[%f,%f]\n", + 0, flux_host(0, 0, 0), flux_expected(0, 0, 0), 1, + flux_host(1, 0, 0), flux_expected(1, 0, 0), 2, flux_host(2, 0, 0), + flux_expected(2, 0, 0), 3, flux_host(3, 0, 0), + flux_expected(3, 0, 0), 4, flux_host(4, 0, 0), + flux_expected(4, 0, 0), 5, flux_host(5, 0, 0), + flux_expected(5, 0, 0)); + REQUIRE(is_close(flux_host(0, 0, 0), flux_expected(0, 0, 0))); + REQUIRE(is_close(flux_host(1, 0, 0), flux_expected(1, 0, 0))); + REQUIRE(is_close(flux_host(2, 0, 0), flux_expected(2, 0, 0))); + REQUIRE(is_close(flux_host(3, 0, 0), flux_expected(3, 0, 0))); + REQUIRE(is_close(flux_host(4, 0, 0), flux_expected(4, 0, 0))); + REQUIRE(is_close(flux_host(5, 0, 0), flux_expected(5, 0, 0))); } + } } TEST_CASE("Test Boundary Handler Struct and Operator") { - // ********************************** Set UP *********************************************************************// - auto lib = Omega_h::Library{}; - auto world = lib.world(); - // simplest 3D mesh - auto mesh = - Omega_h::build_box(world, OMEGA_H_SIMPLEX, 1, 1, 1, 1, 1, 1, false); - printf("[INFO] Mesh created with %d vertices and %d faces\n", - mesh.nverts(), mesh.nfaces()); - - // create particle structure with 5 particles - int num_ptcls = 5; - - // TODO remove this read_write - int argc = 0; - char **argv; - std::string temp_file_name = "mesh.osh"; - Omega_h::binary::write(temp_file_name, &mesh); - fprintf(stdout, "[INFO] Mesh written to file :%s\n", temp_file_name.c_str()); - - - // create particle structure with 5 particles - std::unique_ptr p_pumi_tallyimpl = std::make_unique( - temp_file_name, num_ptcls, - argc, argv); - fprintf(stdout, "[INFO] Particle structure created successfully\n"); - - std::vector init_particle_positions(num_ptcls * 3); - for (int pid = 0; pid < num_ptcls; ++pid) { - init_particle_positions[pid * 3] = 0.1; - init_particle_positions[pid * 3 + 1] = 0.4; - init_particle_positions[pid * 3 + 2] = 0.5; + // ********************************** Set UP + // *********************************************************************// + auto lib = Omega_h::Library{}; + auto world = lib.world(); + // simplest 3D mesh + auto mesh = + Omega_h::build_box(world, OMEGA_H_SIMPLEX, 1, 1, 1, 1, 1, 1, false); + printf("[INFO] Mesh created with %d vertices and %d faces\n", mesh.nverts(), + mesh.nfaces()); + + // create particle structure with 5 particles + int num_ptcls = 5; + + // TODO remove this read_write + int argc = 0; + char **argv; + std::string temp_file_name = "mesh.osh"; + Omega_h::binary::write(temp_file_name, &mesh); + fprintf(stdout, "[INFO] Mesh written to file :%s\n", temp_file_name.c_str()); + + // create particle structure with 5 particles + std::unique_ptr p_pumi_tallyimpl = + std::make_unique(temp_file_name, num_ptcls, + argc, argv); + fprintf(stdout, "[INFO] Particle structure created successfully\n"); + + std::vector init_particle_positions(num_ptcls * 3); + for (int pid = 0; pid < num_ptcls; ++pid) { + init_particle_positions[pid * 3] = 0.1; + init_particle_positions[pid * 3 + 1] = 0.4; + init_particle_positions[pid * 3 + 2] = 0.5; + } + + // this particle structure will be used to check the operator() + p_pumi_tallyimpl->initialize_particle_location( + init_particle_positions.data(), init_particle_positions.size()); + + { + { // *Check if elem_ids_ are 2 now + auto elem_ids_l = p_pumi_tallyimpl->p_particle_tracer_->getElementIds(); + Omega_h::HostRead elem_id_h(elem_ids_l); + for (int pid = 0; pid < num_ptcls; ++pid) { + printf("Element id of particle %d: %d expected %d\n", pid, + elem_id_h[pid], 2); + REQUIRE(elem_id_h[pid] == 2); + } } - // this particle structure will be used to check the operator() - p_pumi_tallyimpl->initialize_particle_location(init_particle_positions.data(), init_particle_positions.size()); - - { - {// *Check if elem_ids_ are 2 now - auto elem_ids_l = p_pumi_tallyimpl->p_particle_tracer_->getElementIds(); - Omega_h::HostRead elem_id_h(elem_ids_l); - for (int pid = 0; pid < num_ptcls; ++pid) { - printf("Element id of particle %d: %d expected %d\n", pid, elem_id_h[pid], 2); - REQUIRE(elem_id_h[pid] == 2); - } - } - - // initialize inter-faces + // initialize inter-faces - // initialize lastExit + // initialize lastExit - {// test prev_xpoint - auto lastExit_l = p_pumi_tallyimpl->p_pumi_particle_at_elem_boundary_handler->prev_xpoint_; - Omega_h::HostWrite lastExit_host(lastExit_l); - REQUIRE(lastExit_host.size() == 3 * p_pumi_tallyimpl->pumipic_ptcls->capacity()); - // prev_xpoints should be the current positions - } + { // test prev_xpoint + auto lastExit_l = + p_pumi_tallyimpl->p_pumi_particle_at_elem_boundary_handler + ->prev_xpoint_; + Omega_h::HostWrite lastExit_host(lastExit_l); + REQUIRE(lastExit_host.size() == + 3 * p_pumi_tallyimpl->pumipic_ptcls->capacity()); + // prev_xpoints should be the current positions + } - // initialize inter_points: should be the initial positions + // initialize inter_points: should be the initial positions - // initialize ptcl_done - Omega_h::Write ptcl_done(num_ptcls, 0, "particle done flag"); + // initialize ptcl_done + Omega_h::Write ptcl_done(num_ptcls, 0, "particle done flag"); - {// * check how inter_points look like - auto inter_points_l = p_pumi_tallyimpl->p_particle_tracer_->getIntersectionPoints(); - Omega_h::HostRead inter_points_host(inter_points_l); - REQUIRE(inter_points_host.size() == p_pumi_tallyimpl->pumipic_ptcls->capacity()*3); // uninitialized - } + { // * check how inter_points look like + auto inter_points_l = + p_pumi_tallyimpl->p_particle_tracer_->getIntersectionPoints(); + Omega_h::HostRead inter_points_host(inter_points_l); + REQUIRE(inter_points_host.size() == + p_pumi_tallyimpl->pumipic_ptcls->capacity() * 3); // uninitialized } - - + } } From 7b9d30d4a912b25c5de26af70ff19cb0fd0ff167 Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Fri, 23 May 2025 16:06:29 -0400 Subject: [PATCH 002/139] particle energy group attribute for flux --- .../pumipic_particle_data_structure.cpp | 97 +++++++++++++------ .../pumipic_particle_data_structure.h | 74 +++++++------- test/test_pumi_tally_impl_methods.cpp | 14 +-- 3 files changed, 113 insertions(+), 72 deletions(-) diff --git a/src/pumitallyopenmc/pumipic_particle_data_structure.cpp b/src/pumitallyopenmc/pumipic_particle_data_structure.cpp index edf3525..0fdd0fa 100644 --- a/src/pumitallyopenmc/pumipic_particle_data_structure.cpp +++ b/src/pumitallyopenmc/pumipic_particle_data_structure.cpp @@ -37,9 +37,9 @@ struct TallyTimes { // ------------------------------------------------------------------------------------------------// // * Data structure for PumiPic // Particle: 0-origin, 1-destination, 2-particle_id, -// 3-in_advance_particle_queue, 4-weight +// 3-in_advance_particle_queue, 4-weight, 5-group typedef pumipic::MemberTypes + Omega_h::I16, Omega_h::Real, int> PPParticle; typedef pumipic::ParticleStructure PPPS; typedef Kokkos::DefaultExecutionSpace PPExeSpace; @@ -91,7 +91,7 @@ struct PumiParticleAtElemBoundary { void finalizeAndWritePumiFlux(Omega_h::Mesh &full_mesh, const std::string &filename); - Omega_h::Reals normalizeFlux(Omega_h::Mesh &mesh); + void normalizeFlux(Omega_h::Mesh &mesh); void mark_initial_as(bool initial); @@ -99,7 +99,7 @@ struct PumiParticleAtElemBoundary { void initialize_flux_array(size_t nelems, size_t nEgroups); bool initial_; // in initial run, flux is not tallied - // Dims: nelems, nEgroups, 2 + // Dims: nelems, nEgroups, 3(flux, flux^2, standard deviation) Kokkos::View flux_; Omega_h::Write prev_xpoint_; Omega_h::Write total_tracklength_; @@ -133,6 +133,7 @@ struct PumiTallyImpl { Omega_h::Write device_pos_buffer_; Omega_h::Write device_in_adv_que_; Omega_h::Write weights_; + Omega_h::Write groups_; TallyTimes tally_times; @@ -160,7 +161,7 @@ struct PumiTallyImpl { int64_t size); void move_to_next_location(double *particle_destinations, int8_t *flying, - double *weights, int64_t size); + double *weights, int *groups, int64_t size); void write_pumi_tally_mesh(); @@ -171,6 +172,8 @@ struct PumiTallyImpl { void copy_and_reset_flying_flag(int8_t *flying); void copy_weights(double *weights); + + void copy_groups(int *groups); }; PumiTallyImpl::PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, @@ -183,6 +186,7 @@ PumiTallyImpl::PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, device_in_adv_que_ = Omega_h::Write(pumi_ps_size, 0, "device_in_adv_que"); weights_ = Omega_h::Write(pumi_ps_size, 0.0, "weights"); + groups_ = Omega_h::Write(pumi_ps_size, 0, "groups"); // todo can track lengths be here? @@ -210,7 +214,7 @@ void PumiTallyImpl::initialize_particle_location( void PumiTallyImpl::move_to_next_location(double *particle_destinations, int8_t *flying, double *weights, - int64_t size) { + int *groups, int64_t size) { assert(size == pumi_ps_size * 3); // copy to device buffer @@ -218,6 +222,7 @@ void PumiTallyImpl::move_to_next_location(double *particle_destinations, // copy fly to device buffer copy_and_reset_flying_flag(flying); copy_weights(weights); + copy_groups(groups); // copy position buffer ps auto particle_dest = pumipic_ptcls->get<1>(); @@ -295,6 +300,24 @@ void PumiTallyImpl::copy_weights(double *weights) { "copy particle weights"); } +void PumiTallyImpl::copy_groups(int *groups) { + auto groups_l = groups_; + Kokkos::View> + host_groups_view(groups, pumi_ps_size); + Kokkos::View> + device_groups_view(groups_l.data(), groups_l.size()); + + Kokkos::deep_copy(device_groups_view, host_groups_view); + auto p_groups = pumipic_ptcls->get<5>(); + auto copy_particle_groups = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + p_groups(pid) = groups_l[pid]; + }; + pumipic::parallel_for(pumipic_ptcls.get(), copy_particle_groups, + "copy particle groups"); +} + void PumiTallyImpl::search_initial_elements() { // assign the location to ptcl // dest auto particle_dest = pumipic_ptcls->get<1>(); @@ -420,11 +443,11 @@ void apply_boundary_condition(Omega_h::Mesh &mesh, PPPS *ptcls, void PumiParticleAtElemBoundary::initialize_flux_array(size_t nelems, size_t nEgroups) { - Kokkos::resize(flux_, nelems, nEgroups, 2); + Kokkos::resize(flux_, nelems, nEgroups, 3); auto flux_l = flux_; OMEGA_H_CHECK(flux_l.extent(0) == nelems); OMEGA_H_CHECK(flux_l.extent(1) == nEgroups); - OMEGA_H_CHECK(flux_l.extent(2) == 2); + OMEGA_H_CHECK(flux_l.extent(2) == 3); } PumiParticleAtElemBoundary::PumiParticleAtElemBoundary(size_t nelems, @@ -498,6 +521,7 @@ void PumiParticleAtElemBoundary::evaluateFlux( auto total_tracklength_l = total_tracklength_; auto in_flight = ptcls->get<3>(); auto p_wgt = ptcls->get<4>(); + auto p_groups = ptcls->get<5>(); auto xpoints_l = xpoints; // todo shouldn't need it, so remove auto evaluate_flux = @@ -531,15 +555,22 @@ void PumiParticleAtElemBoundary::evaluateFlux( Omega_h::Real contribution = segment_length * p_wgt(pid); - Kokkos::atomic_add(&flux(elem_ids[pid], 0, 0), contribution); - Kokkos::atomic_add(&flux(elem_ids[pid], 0, 1), + int group = p_groups(pid); + OMEGA_H_CHECK_PRINTF( + group >= 0 && group < flux.extent(1), + "ERROR: Group %d is out of bounds for flux array with " + "extent %lu\n", + group, flux.extent(1)); + + Kokkos::atomic_add(&flux(elem_ids[pid], group, 0), contribution); + Kokkos::atomic_add(&flux(elem_ids[pid], group, 1), contribution * contribution); } }; pumipic::parallel_for(ptcls, evaluate_flux, "flux evaluation loop"); } -Omega_h::Reals PumiParticleAtElemBoundary::normalizeFlux(Omega_h::Mesh &mesh) { +void PumiParticleAtElemBoundary::normalizeFlux(Omega_h::Mesh &mesh) { const Omega_h::LO nelems = mesh.nelems(); const auto &el2n = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; const auto &coords = mesh.coords(); @@ -549,10 +580,6 @@ Omega_h::Reals PumiParticleAtElemBoundary::normalizeFlux(Omega_h::Mesh &mesh) { Omega_h::Write tet_volumes(flux_.extent(0), -1.0, "tet_volumes"); - Omega_h::Write normalized_flux(flux_.extent(0), -1.0, - "normalized flux"); - Omega_h::Write normalized_flux_2(flux_.extent(0), -1.0, - "normalized flux squared"); Omega_h::Write sd(flux_.extent(0), -1.0, "sd"); auto normalize_flux_with_volume = OMEGA_H_LAMBDA(Omega_h::LO elem_id) { @@ -564,28 +591,37 @@ Omega_h::Reals PumiParticleAtElemBoundary::normalizeFlux(Omega_h::Mesh &mesh) { auto volume = Omega_h::simplex_size_from_basis(b); tet_volumes[elem_id] = volume; - normalized_flux[elem_id] = - flux(elem_id, 0, 0) / (volume * total_tracklength_l.size()); - normalized_flux_2[elem_id] = - flux(elem_id, 0, 1) / (volume * volume * total_tracklength_l.size()); - - // calculate standard deviation FIXME this is not correct, needs number of - // iterations - sd[elem_id] = - Kokkos::sqrt(normalized_flux_2[elem_id] - - normalized_flux[elem_id] * normalized_flux[elem_id]); + for (int g = 0; g < flux.extent(1); g++) { + flux(elem_id, g, 0) /= (volume * total_tracklength_l.size()); + flux(elem_id, g, 1) /= (volume * volume * total_tracklength_l.size()); + + // calculate standard deviation FIXME this is not correct, needs number of + // iterations + flux(elem_id, g, 2) = Kokkos::sqrt( + flux(elem_id, g, 1) - flux(elem_id, g, 0) * flux(elem_id, g, 0)); + } }; Omega_h::parallel_for(tet_volumes.size(), normalize_flux_with_volume, "normalize flux"); mesh.add_tag(Omega_h::REGION, "volume", 1, Omega_h::Reals(tet_volumes)); - return {normalized_flux}; } void PumiParticleAtElemBoundary::finalizeAndWritePumiFlux( Omega_h::Mesh &full_mesh, const std::string &filename) { - const auto &normalized_flux = normalizeFlux(full_mesh); - full_mesh.add_tag(Omega_h::REGION, "flux", 1, normalized_flux); + normalizeFlux(full_mesh); + auto flux = flux_; + int num_groups = flux.extent(1); + Omega_h::Write normalized_flux(flux.extent(0) * num_groups, + 0.0, "normalized_flux"); + // copy flux to normalized_flux + auto copy_flux = OMEGA_H_LAMBDA(Omega_h::LO elem_id) { + for (int g = 0; g < flux.extent(1); g++) { + normalized_flux[elem_id * num_groups + g] = flux(elem_id, g, 0); + } + }; + full_mesh.add_tag(Omega_h::REGION, "flux", num_groups, + Omega_h::Reals(normalized_flux)); Omega_h::vtk::write_parallel(filename, &full_mesh, 3); } @@ -815,10 +851,11 @@ void PumiTally::initialize_particle_location(double *init_particle_positions, void PumiTally::move_to_next_location(double *particle_destinations, int8_t *flying, double *weights, - int64_t size) { + int *groups, int64_t size) { auto start_time = std::chrono::steady_clock::now(); - pimpl->move_to_next_location(particle_destinations, flying, weights, size); + pimpl->move_to_next_location(particle_destinations, flying, weights, groups, + size); std::chrono::duration elapsed_seconds = std::chrono::steady_clock::now() - start_time; diff --git a/src/pumitallyopenmc/pumipic_particle_data_structure.h b/src/pumitallyopenmc/pumipic_particle_data_structure.h index 3537834..1c0e864 100644 --- a/src/pumitallyopenmc/pumipic_particle_data_structure.h +++ b/src/pumitallyopenmc/pumipic_particle_data_structure.h @@ -5,43 +5,45 @@ #ifndef PUMITALLYOPENMC_PUMIPIC_PARTICLE_DATA_STRUCTURE_H #define PUMITALLYOPENMC_PUMIPIC_PARTICLE_DATA_STRUCTURE_H - #include #include namespace pumiinopenmc { - // ------------------------------------------------------------------------------------------------// - // Struct for PumiTallyImpl - struct PumiParticleAtElemBoundary; - struct PumiTallyImpl; - - - // PumiTally class - // This is a pimpl class that contains the implementation of the PumiTallyImpl Struct (Future class) - class PumiTally { - public: - PumiTally(std::string &mesh_filename, int64_t num_particles, int &argc, char **&argv); - - // Public Functions - - // Initialize the particle locations by searching the initial positions of the particles - // in the mesh for the first step - void initialize_particle_location(double *init_particle_positions, int64_t size); - - // Get the new destination of the particles and move the particles to the new destination - void move_to_next_location(double *particle_destinations, int8_t *flying, double *weights, int64_t size); - - // Normalize the tally and write the tally array to vtk file - // Future: send the tally to openmc - void write_pumi_tally_mesh(); - - // Destructor - ~PumiTally(); - - private: - std::unique_ptr pimpl; - }; -} - - -#endif //PUMITALLYOPENMC_PUMIPIC_PARTICLE_DATA_STRUCTURE_H +// ------------------------------------------------------------------------------------------------// +// Struct for PumiTallyImpl +struct PumiParticleAtElemBoundary; +struct PumiTallyImpl; + +// PumiTally class +// This is a pimpl class that contains the implementation of the PumiTallyImpl +// Struct (Future class) +class PumiTally { +public: + PumiTally(std::string &mesh_filename, int64_t num_particles, int &argc, + char **&argv); + + // Public Functions + + // Initialize the particle locations by searching the initial positions of the + // particles in the mesh for the first step + void initialize_particle_location(double *init_particle_positions, + int64_t size); + + // Get the new destination of the particles and move the particles to the new + // destination + void move_to_next_location(double *particle_destinations, int8_t *flying, + double *weights, int *groups, int64_t size); + + // Normalize the tally and write the tally array to vtk file + // Future: send the tally to openmc + void write_pumi_tally_mesh(); + + // Destructor + ~PumiTally(); + +private: + std::unique_ptr pimpl; +}; +} // namespace pumiinopenmc + +#endif // PUMITALLYOPENMC_PUMIPIC_PARTICLE_DATA_STRUCTURE_H diff --git a/test/test_pumi_tally_impl_methods.cpp b/test/test_pumi_tally_impl_methods.cpp index 0dd130b..6f3cf3b 100644 --- a/test/test_pumi_tally_impl_methods.cpp +++ b/test/test_pumi_tally_impl_methods.cpp @@ -110,7 +110,7 @@ TEST_CASE("Test Impl Class Functions") { } //************************************ Checks Regarding Initializing Particle - //Locations ***********************// + // Locations ***********************// //*************************************************************************************************************// { // * Note: all 5 particles will start their journey as follows: @@ -214,9 +214,10 @@ TEST_CASE("Test Impl Class Functions") { { // not a check, just move std::vector flying(num_ptcls, 1); // reset them again to 1 - p_pumi_tallyimpl->move_to_next_location(particle_destination.data(), - flying.data(), weights.data(), - particle_destination.size()); + std::vector group(num_ptcls, 0); + p_pumi_tallyimpl->move_to_next_location( + particle_destination.data(), flying.data(), weights.data(), + group.data(), particle_destination.size()); } { // * Check if the particles correctly reaches element 4 @@ -293,6 +294,7 @@ TEST_CASE("Test Impl Class Functions") { Omega_h::HostWrite next_positions(3 * num_ptcls); std::vector flying_flags(num_ptcls, 0); std::vector particle_weights(num_ptcls, 1); + std::vector group(num_ptcls, 0); for (int pid = 0; pid < num_ptcls; ++pid) { if (pid == 0) { next_positions[3 * pid] = 0.15; @@ -319,7 +321,7 @@ TEST_CASE("Test Impl Class Functions") { } p_pumi_tallyimpl->move_to_next_location( next_positions.data(), flying_flags.data(), particle_weights.data(), - next_positions.size()); + group.data(), next_positions.size()); // ***********************************************************************************************************// { // * check new origins @@ -366,7 +368,7 @@ TEST_CASE("Test Impl Class Functions") { //* particle 3s contributions will go to element 4 and 3 //* particle 5's contribution will go to element 4 only //* segment lengths of 3 are: 0.8790232192610158 and 0.08793076822136835 - //in 4 and 3 respectively + // in 4 and 3 respectively //* segment length of 5 is:b 0.552268050859363 in 4 auto flux_local = p_pumi_tallyimpl->p_pumi_particle_at_elem_boundary_handler->flux_; From a79124f5bf0f57b5d75130752de8ed408f1679fa Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Tue, 17 Jun 2025 13:28:26 -0400 Subject: [PATCH 003/139] copy back material ids --- .../pumipic_particle_data_structure.cpp | 115 +++++++++++++++--- .../pumipic_particle_data_structure.h | 3 +- test/test_pumi_tally_impl_methods.cpp | 6 +- 3 files changed, 105 insertions(+), 19 deletions(-) diff --git a/src/pumitallyopenmc/pumipic_particle_data_structure.cpp b/src/pumitallyopenmc/pumipic_particle_data_structure.cpp index 0fdd0fa..ee82ebb 100644 --- a/src/pumitallyopenmc/pumipic_particle_data_structure.cpp +++ b/src/pumitallyopenmc/pumipic_particle_data_structure.cpp @@ -103,6 +103,7 @@ struct PumiParticleAtElemBoundary { Kokkos::View flux_; Omega_h::Write prev_xpoint_; Omega_h::Write total_tracklength_; + Omega_h::Write material_ids_; }; // ------------------------------------------------------------------------------------------------// @@ -161,7 +162,8 @@ struct PumiTallyImpl { int64_t size); void move_to_next_location(double *particle_destinations, int8_t *flying, - double *weights, int *groups, int64_t size); + double *weights, int *groups, int *material_ids, + int64_t size); void write_pumi_tally_mesh(); @@ -174,6 +176,10 @@ struct PumiTallyImpl { void copy_weights(double *weights); void copy_groups(int *groups); + + void copy_last_location(double *particle_destination, int64_t size); + + void copy_material_ids(int *material_ids, int64_t size); }; PumiTallyImpl::PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, @@ -214,7 +220,8 @@ void PumiTallyImpl::initialize_particle_location( void PumiTallyImpl::move_to_next_location(double *particle_destinations, int8_t *flying, double *weights, - int *groups, int64_t size) { + int *groups, int *material_ids, + int64_t size) { assert(size == pumi_ps_size * 3); // copy to device buffer @@ -252,6 +259,38 @@ void PumiTallyImpl::move_to_next_location(double *particle_destinations, #ifdef PUMI_MEASURE_TIME Kokkos::fence(); #endif + copy_last_location(particle_destinations, size); + copy_material_ids(material_ids, size / 3); +} + +void PumiTallyImpl::copy_last_location(double *particle_destination, + int64_t size) { + auto last_location = p_pumi_particle_at_elem_boundary_handler->prev_xpoint_; + OMEGA_H_CHECK_PRINTF(last_location.size() >= size, + "last_location size %d is not greater to %ld\n", + last_location.size(), size); + Kokkos::View> + device_last_location_view(last_location.data(), size); + Kokkos::View> + host_last_location_view(particle_destination, size); + + Kokkos::deep_copy(host_last_location_view, device_last_location_view); +} + +void PumiTallyImpl::copy_material_ids(int *material_ids, int64_t size) { + auto material_ids_l = p_pumi_particle_at_elem_boundary_handler->material_ids_; + OMEGA_H_CHECK_PRINTF(material_ids_l.size() >= size, + "material_ids_l size %d is not greater to %ld\n", + material_ids_l.size(), size); + Kokkos::View> + host_material_ids_view(material_ids, size); + Kokkos::View> + device_material_ids_view(material_ids_l.data(), size); + + Kokkos::deep_copy(host_material_ids_view, device_material_ids_view); } void PumiTallyImpl::write_pumi_tally_mesh() { @@ -416,19 +455,53 @@ void apply_boundary_condition(Omega_h::Mesh &mesh, PPPS *ptcls, Omega_h::Write &ptcl_done, Omega_h::Write &lastExit, Omega_h::Write &xFace, - Omega_h::Write &inter_points) { + Omega_h::Write &inter_points, + Omega_h::Write material_ids, bool initial) { // TODO: make this a member variable of the struct auto particle_destination = ptcls->get<1>(); + const auto class_ids = mesh.get_array(3, "class_id"); auto checkExposedEdges = PS_LAMBDA(const int e, const int pid, const int mask) { if (mask > 0 && !ptcl_done[pid]) { bool reached_destination = (lastExit[pid] == -1); bool hit_boundary = ((next_elems[pid] == -1) && (elem_ids[pid] != -1)); + + // for the initial run, we need to find the initial position of the + // particles + if (!initial) { // stop at geometry boundary + if (next_elems[pid] != -1) { + if (class_ids[elem_ids[pid]] != + class_ids[next_elems[pid]]) { // particle crosses geometry + // boundary + hit_boundary = true; + material_ids[pid] = class_ids[next_elems[pid]]; + } + } else { + material_ids[pid] = -1; // no material id if not in an element + } + } + ptcl_done[pid] = (reached_destination || hit_boundary) ? 1 : ptcl_done[pid]; + // assert that if the next element is -1, then the material id is -1 + if (!initial) { + if (next_elems[pid] == -1) { + OMEGA_H_CHECK_PRINTF( + material_ids[pid] == -1, + "Error: next_elems[%d] is -1 but material_ids[%d] " + "is %d\n", + pid, pid, material_ids[pid]); + } + // printf("Pid %d next element %d, elem id %d, material id %d\n", + // pid, next_elems[pid], elem_ids[pid], material_ids[pid]); + } if (hit_boundary) { // just reached the boundary + // printf("Moving particle %4d (from -> to): element (%5d -> %5d) + // Material (%3d -> %3d)\n", + // pid, elem_ids[pid], next_elems[pid], + // class_ids[elem_ids[pid]], material_ids[pid]); xFace[pid] = lastExit[pid]; // particle reaches the boundary particle_destination(pid, 0) = inter_points[pid * 3]; @@ -453,8 +526,9 @@ void PumiParticleAtElemBoundary::initialize_flux_array(size_t nelems, PumiParticleAtElemBoundary::PumiParticleAtElemBoundary(size_t nelems, size_t capacity) : prev_xpoint_(capacity * 3, 0.0, "prev_xpoint"), + material_ids_(capacity, -1, "material_ids"), total_tracklength_(capacity, 0.0, "total_tracklength"), initial_(true) { - initialize_flux_array(nelems, 1); + initialize_flux_array(nelems, 2); // FIXME: hardcoded 2 groups printf("[INFO] Particle handler at boundary with %d elements and %d " "x points size (3 * n_particles)\n", flux_.size(), prev_xpoint_.size()); @@ -475,7 +549,8 @@ void PumiParticleAtElemBoundary::operator()( updatePrevXPoint(inter_points); } apply_boundary_condition(mesh, ptcls, elem_ids, next_elems, ptcl_done, - lastExit, inter_faces, inter_points); + lastExit, inter_faces, inter_points, material_ids_, + initial_); move_to_next_element(ptcls, elem_ids, next_elems); } @@ -612,16 +687,20 @@ void PumiParticleAtElemBoundary::finalizeAndWritePumiFlux( normalizeFlux(full_mesh); auto flux = flux_; int num_groups = flux.extent(1); - Omega_h::Write normalized_flux(flux.extent(0) * num_groups, - 0.0, "normalized_flux"); + int num_elems = flux.extent(0); + Omega_h::Write normalized_flux(num_elems, 0.0, + "normalized_flux"); // copy flux to normalized_flux - auto copy_flux = OMEGA_H_LAMBDA(Omega_h::LO elem_id) { - for (int g = 0; g < flux.extent(1); g++) { - normalized_flux[elem_id * num_groups + g] = flux(elem_id, g, 0); - } - }; - full_mesh.add_tag(Omega_h::REGION, "flux", num_groups, - Omega_h::Reals(normalized_flux)); + for (int g = 0; g < num_groups; g++) { + auto copy_flux = OMEGA_H_LAMBDA(Omega_h::LO elem_id) { + normalized_flux[elem_id] = flux(elem_id, g, 0); + }; + Omega_h::parallel_for(num_elems, copy_flux, "copy flux to normalized_flux"); + std::string tag_name = "flux_group_" + std::to_string(g); + Kokkos::fence(); + full_mesh.add_tag(Omega_h::REGION, tag_name, 1, + Omega_h::Reals(normalized_flux)); + } Omega_h::vtk::write_parallel(filename, &full_mesh, 3); } @@ -822,6 +901,9 @@ void PumiTallyImpl::read_pumipic_lib_and_full_mesh(int &argc, char **&argv) { if (full_mesh_.dim() != 3) { printf("PumiPIC only works for 3D mesh now.\n"); } + OMEGA_H_CHECK_PRINTF(full_mesh_.has_tag(Omega_h::REGION, "class_id"), + "Mesh %s does not have class_id tag on regions.\n", + oh_mesh_fname.c_str()); printf("PumiPIC Loaded mesh %s with %d elements\n", oh_mesh_fname.c_str(), full_mesh_.nelems()); } @@ -851,11 +933,12 @@ void PumiTally::initialize_particle_location(double *init_particle_positions, void PumiTally::move_to_next_location(double *particle_destinations, int8_t *flying, double *weights, - int *groups, int64_t size) { + int *groups, int *material_ids, + int64_t size) { auto start_time = std::chrono::steady_clock::now(); pimpl->move_to_next_location(particle_destinations, flying, weights, groups, - size); + material_ids, size); std::chrono::duration elapsed_seconds = std::chrono::steady_clock::now() - start_time; diff --git a/src/pumitallyopenmc/pumipic_particle_data_structure.h b/src/pumitallyopenmc/pumipic_particle_data_structure.h index 1c0e864..e5cd6b5 100644 --- a/src/pumitallyopenmc/pumipic_particle_data_structure.h +++ b/src/pumitallyopenmc/pumipic_particle_data_structure.h @@ -32,7 +32,8 @@ class PumiTally { // Get the new destination of the particles and move the particles to the new // destination void move_to_next_location(double *particle_destinations, int8_t *flying, - double *weights, int *groups, int64_t size); + double *weights, int *groups, int *material_ids, + int64_t size); // Normalize the tally and write the tally array to vtk file // Future: send the tally to openmc diff --git a/test/test_pumi_tally_impl_methods.cpp b/test/test_pumi_tally_impl_methods.cpp index 6f3cf3b..89057c1 100644 --- a/test/test_pumi_tally_impl_methods.cpp +++ b/test/test_pumi_tally_impl_methods.cpp @@ -215,9 +215,10 @@ TEST_CASE("Test Impl Class Functions") { { // not a check, just move std::vector flying(num_ptcls, 1); // reset them again to 1 std::vector group(num_ptcls, 0); + std::vector materials_ids(num_ptcls, 0); // not used p_pumi_tallyimpl->move_to_next_location( particle_destination.data(), flying.data(), weights.data(), - group.data(), particle_destination.size()); + group.data(), materials_ids.data(), particle_destination.size()); } { // * Check if the particles correctly reaches element 4 @@ -295,6 +296,7 @@ TEST_CASE("Test Impl Class Functions") { std::vector flying_flags(num_ptcls, 0); std::vector particle_weights(num_ptcls, 1); std::vector group(num_ptcls, 0); + std::vector materials_ids(num_ptcls, 0); // not used for (int pid = 0; pid < num_ptcls; ++pid) { if (pid == 0) { next_positions[3 * pid] = 0.15; @@ -321,7 +323,7 @@ TEST_CASE("Test Impl Class Functions") { } p_pumi_tallyimpl->move_to_next_location( next_positions.data(), flying_flags.data(), particle_weights.data(), - group.data(), next_positions.size()); + group.data(), materials_ids.data(), next_positions.size()); // ***********************************************************************************************************// { // * check new origins From afc0f21c87dcbeb26bf43f8322015a02ae2a9176 Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Fri, 20 Jun 2025 11:12:02 -0400 Subject: [PATCH 004/139] degas2 physics api --- CMakeLists.txt | 41 ++++++++++++++++++ src/physics/degas2/DG2CrossSection.cpp | 9 ++++ src/physics/degas2/DG2CrossSection.h | 32 ++++++++++++++ src/physics/degas2/DG2Physics.cpp | 16 +++++++ src/physics/degas2/DG2Physics.h | 59 ++++++++++++++++++++++++++ 5 files changed, 157 insertions(+) create mode 100644 src/physics/degas2/DG2CrossSection.cpp create mode 100644 src/physics/degas2/DG2CrossSection.h create mode 100644 src/physics/degas2/DG2Physics.cpp create mode 100644 src/physics/degas2/DG2Physics.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 10a2a87..2534d64 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,7 @@ option(BUILD_SHARED_LIBS "Build shared libraries" ON) option(PUMITALLYOPENMC_ENABLE_TESTS "Enable tests" ON) option(PUMI_USE_KOKKOS_CUDA "Use CUDA" OFF) option(PUMI_MEASURE_TIME "Measure time" OFF) +option(PUMI_ENABLE_DEGAS2 "Enable Degas2" ON) ################################################################### ############################ CLANG ################################ @@ -33,6 +34,18 @@ set(PUMITALLYOPENMC_SOURCES src/pumitallyopenmc/pumipic_particle_data_structure.cpp ) +if(PUMI_ENABLE_DEGAS2) + set(PUMI_DEGAS2_HEADERS + src/physics/degas2/DG2Physics.h + src/physics/degas2/DG2CrossSection.h + ) + set(PUMI_DEGAS2_SOURCES + src/physics/degas2/DG2Physics.cpp + src/physics/degas2/DG2CrossSection.cpp + ) +endif() + + ################################## Testing ######################## if(PUMITALLYOPENMC_ENABLE_TESTS) @@ -76,6 +89,14 @@ if(PUMI_MEASURE_TIME) target_compile_definitions(pumitallyopenmc PRIVATE -DPUMI_MEASURE_TIME) endif() + +if(PUMI_ENABLE_DEGAS2) + find_package(HDF5 REQUIRED COMPONENTS CXX) + add_library(pumitally_degas2) + target_sources(pumitally_degas2 PRIVATE ${PUMI_DEGAS2_SOURCES}) + target_link_libraries(pumitally_degas2 PUBLIC Omega_h::omega_h HDF5::HDF5) + target_include_directories(pumitally_degas2 PUBLIC ${HDF5_INCLUDE_DIRS}) +endif() ################################################################### @@ -140,5 +161,25 @@ install(FILES DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pumitallyopenmc ) +if(PUMI_ENABLE_DEGAS2) + install(TARGETS pumitally_degas2 + EXPORT pumitallyopenmcTargets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/pumitallyopenmc + ) + install(EXPORT pumitallyopenmcTargets + FILE pumitally_degas2Targets.cmake + NAMESPACE pumitallyopenmc:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/pumitallyopenmc + ) + install(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/src/physics/degas2/DG2Physics.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/pumitallyopenmc/physics/degas2 + ) +endif() + ################################################################### diff --git a/src/physics/degas2/DG2CrossSection.cpp b/src/physics/degas2/DG2CrossSection.cpp new file mode 100644 index 0000000..4b3be41 --- /dev/null +++ b/src/physics/degas2/DG2CrossSection.cpp @@ -0,0 +1,9 @@ +// +// Created by hasanm4 on 6/20/25. +// + +#include "DG2CrossSection.h" + +DG2CrossSection::DG2CrossSection(std::string filename) { + // fill in the arrays +} diff --git a/src/physics/degas2/DG2CrossSection.h b/src/physics/degas2/DG2CrossSection.h new file mode 100644 index 0000000..fa14e30 --- /dev/null +++ b/src/physics/degas2/DG2CrossSection.h @@ -0,0 +1,32 @@ +// +// Created by hasanm4 on 6/20/25. +// + +#ifndef PUMITALLYOPENMC_DG2CROSSSECTION_H +#define PUMITALLYOPENMC_DG2CROSSSECTION_H + +#include +#include + +class DG2CrossSection { +public: + // fills the cross-section arrays from the given file + explicit DG2CrossSection(std::string filename); + + // Getters go here + + // no need to define readers of the cross-section arrays + +private: + std::string sourceFileName_; + int nEnergyGroups_; + int nMaterials_; + int nTemperatures_; + + Kokkos::View sigma_t_; // mat, T, g + Kokkos::View sigma_a_; // mat, T, g + Kokkos::View scattering_matrix_; // mat, T, g, g + Kokkos::View sigma_s_; // mat, T, g +}; + +#endif // PUMITALLYOPENMC_DG2CROSSSECTION_H diff --git a/src/physics/degas2/DG2Physics.cpp b/src/physics/degas2/DG2Physics.cpp new file mode 100644 index 0000000..d9454cd --- /dev/null +++ b/src/physics/degas2/DG2Physics.cpp @@ -0,0 +1,16 @@ +// +// Created by Fuad Hasan on 6/20/25. +// +#include "DG2Physics.h" + +KOKKOS_FUNCTION +void DG2Physics::sample_collision_distance(ParticleInfo &particle_info, + const FieldInfo &field_info) { + // update particle info based on the physics +} + +KOKKOS_FUNCTION +void DG2Physics::collide_particle(ParticleInfo &particle_info, + const FieldInfo &field_info) { + // Handle the collision of a particle with the field +} diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h new file mode 100644 index 0000000..1920fc7 --- /dev/null +++ b/src/physics/degas2/DG2Physics.h @@ -0,0 +1,59 @@ +// +// Created by Fuad Hasan on 6/20/25. +// + +#ifndef PUMITALLYOPENMC_DG2PHYSICS_H +#define PUMITALLYOPENMC_DG2PHYSICS_H + +#define SEED 12345 + +#include "DG2CrossSection.h" +#include +#include + +typedef Kokkos::Random_XorShift64_Pool + random_pool_t; + +struct ParticleInfo { + double position[3]; // Position in space (x, y, z) + double direction[3]; // Direction vector (unit vector) + double weight; + int energy_group; // Energy group *index* +}; + +struct FieldInfo { + double electron_temperature; + double ion_temperature; + double electron_density; + double ion_density; + double bulk_flow_velocity[3]; +}; + +class DG2Physics { +public: + DG2Physics(const std::string &cross_section_file, const int num_particles, + const int seed = SEED) + : random_pool_(seed), cross_section_(cross_section_file) { + // Initialize particle velocities + particle_velocities_ = + Kokkos::View("particle_velocities", num_particles); + } + + // sample a new distance and update the particle's position but does not + // change direction + KOKKOS_FUNCTION + void sample_collision_distance(ParticleInfo &particle_info, + const FieldInfo &field_info); + + // collision event + KOKKOS_FUNCTION + void collide_particle(ParticleInfo &particle_info, + const FieldInfo &field_info); + +private: + random_pool_t random_pool_; + DG2CrossSection cross_section_; + + Kokkos::View particle_velocities_; +}; +#endif // PUMITALLYOPENMC_DG2PHYSICS_H From a0ea9f393fdd6c9c47cf453ceed0b26dc0f185b4 Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Fri, 20 Jun 2025 16:45:56 -0400 Subject: [PATCH 005/139] degas2 physics test case --- src/physics/degas2/DG2CrossSection.cpp | 2 +- src/physics/degas2/DG2CrossSection.h | 29 ++++++++------ src/physics/degas2/DG2Physics.cpp | 12 ------ src/physics/degas2/DG2Physics.h | 32 +++++++++++----- test/CMakeLists.txt | 7 ++++ test/test_degas2_physics.cpp | 53 ++++++++++++++++++++++++++ 6 files changed, 102 insertions(+), 33 deletions(-) create mode 100644 test/test_degas2_physics.cpp diff --git a/src/physics/degas2/DG2CrossSection.cpp b/src/physics/degas2/DG2CrossSection.cpp index 4b3be41..b83745a 100644 --- a/src/physics/degas2/DG2CrossSection.cpp +++ b/src/physics/degas2/DG2CrossSection.cpp @@ -1,5 +1,5 @@ // -// Created by hasanm4 on 6/20/25. +// Created by Fuad Hasans on 6/20/25. // #include "DG2CrossSection.h" diff --git a/src/physics/degas2/DG2CrossSection.h b/src/physics/degas2/DG2CrossSection.h index fa14e30..9602214 100644 --- a/src/physics/degas2/DG2CrossSection.h +++ b/src/physics/degas2/DG2CrossSection.h @@ -1,5 +1,5 @@ // -// Created by hasanm4 on 6/20/25. +// Created by Fuad Hasan on 6/20/25. // #ifndef PUMITALLYOPENMC_DG2CROSSSECTION_H @@ -13,20 +13,27 @@ class DG2CrossSection { // fills the cross-section arrays from the given file explicit DG2CrossSection(std::string filename); - // Getters go here + // explicitly given cross-section arrays constructor + DG2CrossSection(Kokkos::View sigma_t, + Kokkos::View sigma_a, + Kokkos::View scattering_matrix, + Kokkos::View sigma_s) + : nEnergyGroups(sigma_t.extent(2)), nMaterials(sigma_t.extent(0)), + nTemperatures(sigma_t.extent(1)), sigma_t(sigma_t), sigma_a(sigma_a), + scattering_matrix(scattering_matrix), sigma_s(sigma_s) {} + + DG2CrossSection(const DG2CrossSection &other) = default; // no need to define readers of the cross-section arrays -private: - std::string sourceFileName_; - int nEnergyGroups_; - int nMaterials_; - int nTemperatures_; + int nEnergyGroups; + int nMaterials; + int nTemperatures; - Kokkos::View sigma_t_; // mat, T, g - Kokkos::View sigma_a_; // mat, T, g - Kokkos::View scattering_matrix_; // mat, T, g, g - Kokkos::View sigma_s_; // mat, T, g + Kokkos::View sigma_t; // mat, T, g + Kokkos::View sigma_a; // mat, T, g + Kokkos::View scattering_matrix; // mat, T, g, g + Kokkos::View sigma_s; // mat, T, g }; #endif // PUMITALLYOPENMC_DG2CROSSSECTION_H diff --git a/src/physics/degas2/DG2Physics.cpp b/src/physics/degas2/DG2Physics.cpp index d9454cd..4b986b7 100644 --- a/src/physics/degas2/DG2Physics.cpp +++ b/src/physics/degas2/DG2Physics.cpp @@ -2,15 +2,3 @@ // Created by Fuad Hasan on 6/20/25. // #include "DG2Physics.h" - -KOKKOS_FUNCTION -void DG2Physics::sample_collision_distance(ParticleInfo &particle_info, - const FieldInfo &field_info) { - // update particle info based on the physics -} - -KOKKOS_FUNCTION -void DG2Physics::collide_particle(ParticleInfo &particle_info, - const FieldInfo &field_info) { - // Handle the collision of a particle with the field -} diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 1920fc7..72f45be 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -33,9 +33,17 @@ class DG2Physics { public: DG2Physics(const std::string &cross_section_file, const int num_particles, const int seed = SEED) - : random_pool_(seed), cross_section_(cross_section_file) { + : random_pool(seed), cross_section(cross_section_file) { // Initialize particle velocities - particle_velocities_ = + particle_velocities = + Kokkos::View("particle_velocities", num_particles); + } + + DG2Physics(const DG2CrossSection cross_section, const int num_particles, + const int seed = SEED) + : random_pool(seed), cross_section(cross_section) { + // Initialize particle velocities + particle_velocities = Kokkos::View("particle_velocities", num_particles); } @@ -43,17 +51,23 @@ class DG2Physics { // change direction KOKKOS_FUNCTION void sample_collision_distance(ParticleInfo &particle_info, - const FieldInfo &field_info); + const FieldInfo &field_info) const { + // example of sampling a random number + auto rand_gen = random_pool.get_state(); + double x = rand_gen.drand(0., 1.); + random_pool.free_state(rand_gen); + + double y = + x * cross_section.sigma_a(0, 0, 0); // Example usage of cross_section + } // collision event KOKKOS_FUNCTION void collide_particle(ParticleInfo &particle_info, - const FieldInfo &field_info); - -private: - random_pool_t random_pool_; - DG2CrossSection cross_section_; + const FieldInfo &field_info) const {} + random_pool_t random_pool; + DG2CrossSection cross_section; - Kokkos::View particle_velocities_; + Kokkos::View particle_velocities; }; #endif // PUMITALLYOPENMC_DG2PHYSICS_H diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 86f3074..4f4398b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,4 +28,11 @@ add_executable(test_pumi_tally_impl_methods test_pumi_tally_impl_methods.cpp) target_include_directories(test_pumi_tally_impl_methods PRIVATE ${PROJECT_SOURCE_DIR}/src/pumitallyopenmc) target_link_libraries(test_pumi_tally_impl_methods PUBLIC Catch2::Catch2WithMain pumipic::pumipic) catch_discover_tests(test_pumi_tally_impl_methods) + +if(PUMI_ENABLE_DEGAS2) + add_executable(test_degas2_physics test_degas2_physics.cpp) + target_include_directories(test_degas2_physics PRIVATE ${PROJECT_SOURCE_DIR}/src/physics/degas2) + target_link_libraries(test_degas2_physics PUBLIC Catch2::Catch2 pumitally_degas2) + catch_discover_tests(test_degas2_physics) +endif() #################### Test HostWrite #################### \ No newline at end of file diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp new file mode 100644 index 0000000..4fa4d02 --- /dev/null +++ b/test/test_degas2_physics.cpp @@ -0,0 +1,53 @@ +// +// Created by Fuad Hasan on 6/20/25. +// + +#include +#include + +#include "DG2Physics.h" +#include + +int main(int argc, char *argv[]) { + Kokkos::initialize(argc, argv); + + { + int numParticles = 10; + + Kokkos::View sigma_t_; // mat, T, g + Kokkos::View sigma_a_; // mat, T, g + Kokkos::View scattering_matrix_; // mat, T, g, g + Kokkos::View sigma_s_; // mat, T, g + DG2CrossSection crossSection(sigma_t_, sigma_a_, scattering_matrix_, + sigma_s_); + DG2Physics physics(crossSection, numParticles); + + Kokkos::View particles("particles", numParticles); + Kokkos::View fields("fields", numParticles); + Kokkos::parallel_for( + "initialize_particles", numParticles, KOKKOS_LAMBDA(int i) { + particles(i).position[0] = 0; + particles(i).position[1] = 0; + particles(i).position[2] = 0; + particles(i).direction[0] = 1; + particles(i).direction[1] = 0; + particles(i).direction[2] = 0; + particles(i).energy_group = 1; + particles(i).weight = 1.0; + + fields(i).electron_density = 1.0; + fields(i).ion_density = 1.0; + fields(i).electron_temperature = 1.0; + fields(i).ion_temperature = 1.0; + // and others like this + }); + + // Call the function to be tested + Kokkos::parallel_for( + "run physics", numParticles, KOKKOS_LAMBDA(int i) { + physics.collide_particle(particles(i), fields(i)); + }); + } + Kokkos::finalize(); + return 0; +} From fe4548e76975a32496b2b791ec6b22e76efaa899 Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Wed, 25 Jun 2025 15:53:43 -0400 Subject: [PATCH 006/139] test discovery at pre test instead of during build --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4f4398b..f5c1995 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -33,6 +33,6 @@ if(PUMI_ENABLE_DEGAS2) add_executable(test_degas2_physics test_degas2_physics.cpp) target_include_directories(test_degas2_physics PRIVATE ${PROJECT_SOURCE_DIR}/src/physics/degas2) target_link_libraries(test_degas2_physics PUBLIC Catch2::Catch2 pumitally_degas2) - catch_discover_tests(test_degas2_physics) + catch_discover_tests(test_degas2_physics DISCOVERY_MODE PRE_TEST) endif() #################### Test HostWrite #################### \ No newline at end of file From a6af7861969b6d1bd98af5733a3ba41fd02efaaa Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 30 Jun 2025 14:33:42 -0400 Subject: [PATCH 007/139] Add ionization and charge exchange roughly --- src/physics/degas2/DG2Physics.h | 196 +++++++++++++++++++++++++++++++- 1 file changed, 192 insertions(+), 4 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 72f45be..9428986 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -41,14 +41,25 @@ class DG2Physics { DG2Physics(const DG2CrossSection cross_section, const int num_particles, const int seed = SEED) + : random_pool(seed), cross_section(cross_section) { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); } - + //From Here // sample a new distance and update the particle's position but does not // change direction + /* + //Functions to evaluate the cross sections. Need to talk to Fuad + //For now I'll put them in the functions + KOKKOS_FUNCTION + double ionization_cross_section(ParticleInfo &particle_info, + const FieldInfo &field_info){ + + } + */ + KOKKOS_FUNCTION void sample_collision_distance(ParticleInfo &particle_info, const FieldInfo &field_info) const { @@ -57,14 +68,191 @@ class DG2Physics { double x = rand_gen.drand(0., 1.); random_pool.free_state(rand_gen); - double y = - x * cross_section.sigma_a(0, 0, 0); // Example usage of cross_section + //Compute Ionization Cross Section + double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 + double mag2 {2*par.energy_group/mp}; //cm^2/s^2 + + double coef[9]; + + coef[0] = -3.271396786375e1; coef[1] = 1.353655609057e1; coef[2] = -5.739328757388; + coef[3] = 1.563154982022; coef[4] = -2.877056004391e-1; coef[5] = 3.482559773737e-2; + coef[6] = -2.631976175590e-3; coef[7] = 1.119543953861e-4; coef[8] = -2.039149852002e-6; + + double lnsigma_ion {0}; //cm^2 + for (int i=0; i<9; i++) { + lnsigma_ion += coef[i]*std::pow(std::log(fie.electron_temperature),i); + } + double sigma_ion {exp(lnsigma_ion)/sqrt(mag2)}; + + //Compute Charge Exchange Cross Section + double coef [9][9]; //E index, T index + + coef[0][0] = -1.829079581680e1; coef[0][1] = 2.169137615703e-1; coef[0][2] = 4.307131243894e-2; + coef[0][3] = -5.754895093075e-4; coef[0][4] = -1.552077120204e-3; coef[0][5] = -1.876800283030e-4; + coef[0][6] = 1.125490270962e-4; coef[0][7] = -1.238982763007e-5; coef[0][8] = 4.163596197181e-07; + + coef[1][0] = 1.640252721210e-1; coef[1][1] = -1.106722014459e-1; coef[1][2] = 8.948693624917e-3; + coef[1][3] = 6.062141761233e-3; coef[1][4] = -1.210431587568e-3; coef[1][5] = -4.052878751584e-5; + coef[1][6] = 2.875900435895e-5; coef[1][7] = -2.616998139678e-6; coef[1][8] = 7.558092849125e-8; + + coef[2][0] = 3.364564509137e-2; coef[2][1] = -1.382158680424e-3; coef[2][2] = -1.209480567154e-02; + coef[2][3] = 1.075907881928e-3; coef[2][4] = 8.297212635856e-4; coef[2][5] = -1.907025662962e-04; + coef[2][6] = 1.338839628570e-5; coef[2][7] = -1.171762874107e-7; coef[2][8] = -1.328404104165e-8; + + coef[3][0] = 9.530225559189e-3; coef[3][1] = 7.348786286628e-3; coef[3][2] = -3.675019470470e-4; + coef[3][3] = -8.119301728339e-4; coef[3][4] = 1.361661816974e-4; coef[3][5] = 1.141663041636e-5; + coef[3][6] = -4.340802793033e-6; coef[3][7] = 3.517971869029e-7; coef[3][8] = -9.170850253981e-09; + + coef[4][0] = -8.519413589968e-4; coef[4][1] = -6.343059502294e-4; coef[4][2] = 1.039643390686e-3; + coef[4][3] = 8.911036876068e-6; coef[4][4] = -1.008928628425e-4; coef[4][5] = 1.775681984457e-05; + coef[4][6] = -7.003521917385e-7; coef[4][7] = -4.928692832866e-8; coef[4][8] = 3.208853883734e-9; + + coef[5][0] = -1.247583860943e-3; coef[5][1] = -1.919569450380e-4; coef[5][2] = -1.553840717902e-4; + coef[5][3] = 3.175388949811e-5; coef[5][4] = 1.080693990468e-5; coef[5][5] = -3.149286923815e-6; + coef[5][6] = 2.318308730487e-7; coef[5][7] = 1.756388998863e-10; coef[5][8] = -3.952740758950e-10; + + coef[6][0] = 3.014307545716e-4; coef[6][1] = 4.075019351738e-5; coef[6][2] = 2.670827249272e-6; + coef[6][3] = -4.515123641755e-6; coef[6][4] = 5.106059413591e-7; coef[6][5] = 3.105491554749e-8; + coef[6][6] = -6.030983538280e-9; coef[6][7] = -1.446756795654e-10; coef[6][8] = 2.739558475782e-11; + + coef[7][0] = -2.499323170044e-5; coef[7][1] = -2.850044983009e-6; coef[7][2] = 7.695300597935e-7; + coef[7][3] = 2.187439283954e-7; coef[7][4] = -1.299275586093e-7; coef[7][5] = 2.274394089017e-8; + coef[7][6] = -1.755944926274e-9; coef[7][7] = 7.143183138281e-11; coef[7][8] = -1.693040208927e-12; + + coef[8][0] = 6.932627237765e-7; coef[8][1] = 6.966822400446e-8; coef[8][2] = -3.783302281524e-8; + coef[8][3] = -2.911233951880e-9; coef[8][4] = 5.117133050290e-9; coef[8][5] = -1.130988250912e-9; + coef[8][6] = 1.005189187279e-10; coef[8][7] = -3.989884105603e-12; coef[8][8] = 6.388219930167e-14; + + double lnalpha {0}; + for (int i = 0; i < 9; i++) { + for (int j = 0; j < 9; j++) { + lnalpha += coef[i][j]*std::pow(std::log(par.energy_group),i)*std::pow(std::log(fie.ion_temperature),j); + } + } + double sigma_cx = std::exp(lnalpha)/sqrt(mag2); + + //Generate distance and move particle + double l =-std::log(x)/(fie.electron_density*sigma_ion+fie.ion_density*sigma_cx); //cm. n in cm^-3 + particle_info.position[0] += l*particle_info.direction[0]; + particle_info.position[1] += l*particle_info.direction[1]; + particle_info.position[2] += l*particle_info.direction[2]; } // collision event KOKKOS_FUNCTION void collide_particle(ParticleInfo &particle_info, - const FieldInfo &field_info) const {} + const FieldInfo &field_info) const { + + //Generate Random Numbers + auto rand_gen = random_pool.get_state(); + double x1 = rand_gen.drand(0., 1.); + double x2 = rand_gen.drand(0., 1.); + double y1 = rand_gen.drand(0., 1.); + double y2 = rand_gen.drand(0., 1.); + double ww = rand_gen.drand(0., 1.); + random_pool.free_state(rand_gen); + + //Compute Ionization Cross Section + double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 + double mag2 {2*par.energy_group/mp}; //cm^2/s^2 + + double coef[9]; + + coef[0] = -3.271396786375e1; coef[1] = 1.353655609057e1; coef[2] = -5.739328757388; + coef[3] = 1.563154982022; coef[4] = -2.877056004391e-1; coef[5] = 3.482559773737e-2; + coef[6] = -2.631976175590e-3; coef[7] = 1.119543953861e-4; coef[8] = -2.039149852002e-6; + + double lnsigma_ion {0}; //cm^2 + for (int i=0; i<9; i++) { + lnsigma_ion += coef[i]*std::pow(std::log(fie.electron_temperature),i); + } + double sigma_ion {exp(lnsigma_ion)/sqrt(mag2)}; + + //Compute Charge Exchange Cross Section + double coef [9][9]; //E index, T index + + coef[0][0] = -1.829079581680e1; coef[0][1] = 2.169137615703e-1; coef[0][2] = 4.307131243894e-2; + coef[0][3] = -5.754895093075e-4; coef[0][4] = -1.552077120204e-3; coef[0][5] = -1.876800283030e-4; + coef[0][6] = 1.125490270962e-4; coef[0][7] = -1.238982763007e-5; coef[0][8] = 4.163596197181e-07; + + coef[1][0] = 1.640252721210e-1; coef[1][1] = -1.106722014459e-1; coef[1][2] = 8.948693624917e-3; + coef[1][3] = 6.062141761233e-3; coef[1][4] = -1.210431587568e-3; coef[1][5] = -4.052878751584e-5; + coef[1][6] = 2.875900435895e-5; coef[1][7] = -2.616998139678e-6; coef[1][8] = 7.558092849125e-8; + + coef[2][0] = 3.364564509137e-2; coef[2][1] = -1.382158680424e-3; coef[2][2] = -1.209480567154e-02; + coef[2][3] = 1.075907881928e-3; coef[2][4] = 8.297212635856e-4; coef[2][5] = -1.907025662962e-04; + coef[2][6] = 1.338839628570e-5; coef[2][7] = -1.171762874107e-7; coef[2][8] = -1.328404104165e-8; + + coef[3][0] = 9.530225559189e-3; coef[3][1] = 7.348786286628e-3; coef[3][2] = -3.675019470470e-4; + coef[3][3] = -8.119301728339e-4; coef[3][4] = 1.361661816974e-4; coef[3][5] = 1.141663041636e-5; + coef[3][6] = -4.340802793033e-6; coef[3][7] = 3.517971869029e-7; coef[3][8] = -9.170850253981e-09; + + coef[4][0] = -8.519413589968e-4; coef[4][1] = -6.343059502294e-4; coef[4][2] = 1.039643390686e-3; + coef[4][3] = 8.911036876068e-6; coef[4][4] = -1.008928628425e-4; coef[4][5] = 1.775681984457e-05; + coef[4][6] = -7.003521917385e-7; coef[4][7] = -4.928692832866e-8; coef[4][8] = 3.208853883734e-9; + + coef[5][0] = -1.247583860943e-3; coef[5][1] = -1.919569450380e-4; coef[5][2] = -1.553840717902e-4; + coef[5][3] = 3.175388949811e-5; coef[5][4] = 1.080693990468e-5; coef[5][5] = -3.149286923815e-6; + coef[5][6] = 2.318308730487e-7; coef[5][7] = 1.756388998863e-10; coef[5][8] = -3.952740758950e-10; + + coef[6][0] = 3.014307545716e-4; coef[6][1] = 4.075019351738e-5; coef[6][2] = 2.670827249272e-6; + coef[6][3] = -4.515123641755e-6; coef[6][4] = 5.106059413591e-7; coef[6][5] = 3.105491554749e-8; + coef[6][6] = -6.030983538280e-9; coef[6][7] = -1.446756795654e-10; coef[6][8] = 2.739558475782e-11; + + coef[7][0] = -2.499323170044e-5; coef[7][1] = -2.850044983009e-6; coef[7][2] = 7.695300597935e-7; + coef[7][3] = 2.187439283954e-7; coef[7][4] = -1.299275586093e-7; coef[7][5] = 2.274394089017e-8; + coef[7][6] = -1.755944926274e-9; coef[7][7] = 7.143183138281e-11; coef[7][8] = -1.693040208927e-12; + + coef[8][0] = 6.932627237765e-7; coef[8][1] = 6.966822400446e-8; coef[8][2] = -3.783302281524e-8; + coef[8][3] = -2.911233951880e-9; coef[8][4] = 5.117133050290e-9; coef[8][5] = -1.130988250912e-9; + coef[8][6] = 1.005189187279e-10; coef[8][7] = -3.989884105603e-12; coef[8][8] = 6.388219930167e-14; + + double lnalpha {0}; + for (int i = 0; i < 9; i++) { + for (int j = 0; j < 9; j++) { + lnalpha += coef[i][j]*std::pow(std::log(par.energy_group),i)*std::pow(std::log(fie.ion_temperature),j); + } + } + double sigma_cx = std::exp(lnalpha)/sqrt(mag2); + + //Compute New Direction and Energy and set particle info + //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) + auto xx = std::sqrt(-2*std::log(x1))*std::cos(2*std::numbers::pi*x2); + auto yy = std::sqrt(-2*std::log(x1))*std::sin(2*std::numbers::pi*x2); + auto zz = std::sqrt(-2*std::log(y1))*std::cos(2*std::numbers::pi*y2); + + auto ux = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(xx))*cos(2*std::numbers::pi*y); + auto uy = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(yy))*sin(2*std::numbers::pi*y); + auto uz = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(zz))*sin(2*std::numbers::pi*y); + + auto mag_u = std::sqrt(ux*ux + uy*uy + uz*uz); + + particle_info.direction[0] = ux/mag_u; + particle_info.direction[1] = uy/mag_u; + particle_info.direction[2] = uz/mag_u; + + particle_info.energy_group = 0.5*mp*mag_u*mag_u; + + //Adjust Weights + double new_weight = particle_info.weight*(1- + (field_info.electron_density*sigma_ion)/ + (field_info.ion_density*sigma_cx+field_info.electron_density*sigma_ion)); + + //Russian Roulette + double wc = 0.25; + double ws = 1.0; + + if (new_weight < wc){ + if (ww<(1-new_weight/ws)) { + new_weight = 0; + } + else { + new_weight = ws; + } + } + particle_info.weight = new_weight; +} +//To here random_pool_t random_pool; DG2CrossSection cross_section; From 5793685ee5fa2f1f607f630aac83a41c23fae6db Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 30 Jun 2025 15:52:16 -0400 Subject: [PATCH 008/139] Fixed mislabelled variables, std::num --- CMakeLists.txt | 2 +- src/physics/degas2/DG2Physics.h | 35 ++++++++++++++++----------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2534d64..0b35577 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.18) project(pumitallyopenmc) include(FetchContent) include(GNUInstallDirs) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 9428986..dec3a65 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -63,14 +63,15 @@ class DG2Physics { KOKKOS_FUNCTION void sample_collision_distance(ParticleInfo &particle_info, const FieldInfo &field_info) const { - // example of sampling a random number + using namespace std::numbers + // example of sampling a random number auto rand_gen = random_pool.get_state(); double x = rand_gen.drand(0., 1.); random_pool.free_state(rand_gen); //Compute Ionization Cross Section double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double mag2 {2*par.energy_group/mp}; //cm^2/s^2 + double mag2 {2*particle_info.energy_group/mp}; //cm^2/s^2 double coef[9]; @@ -80,7 +81,7 @@ class DG2Physics { double lnsigma_ion {0}; //cm^2 for (int i=0; i<9; i++) { - lnsigma_ion += coef[i]*std::pow(std::log(fie.electron_temperature),i); + lnsigma_ion += coef[i]*std::pow(std::log(field_info.electron_temperature),i); } double sigma_ion {exp(lnsigma_ion)/sqrt(mag2)}; @@ -126,13 +127,13 @@ class DG2Physics { double lnalpha {0}; for (int i = 0; i < 9; i++) { for (int j = 0; j < 9; j++) { - lnalpha += coef[i][j]*std::pow(std::log(par.energy_group),i)*std::pow(std::log(fie.ion_temperature),j); + lnalpha += coef[i][j]*std::pow(std::log(particle_info.energy_group),i)*std::pow(std::log(field_info.ion_temperature),j); } } double sigma_cx = std::exp(lnalpha)/sqrt(mag2); //Generate distance and move particle - double l =-std::log(x)/(fie.electron_density*sigma_ion+fie.ion_density*sigma_cx); //cm. n in cm^-3 + double l =-std::log(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 particle_info.position[0] += l*particle_info.direction[0]; particle_info.position[1] += l*particle_info.direction[1]; particle_info.position[2] += l*particle_info.direction[2]; @@ -143,6 +144,7 @@ class DG2Physics { void collide_particle(ParticleInfo &particle_info, const FieldInfo &field_info) const { + using namespace std::numbers //Generate Random Numbers auto rand_gen = random_pool.get_state(); double x1 = rand_gen.drand(0., 1.); @@ -154,17 +156,17 @@ class DG2Physics { //Compute Ionization Cross Section double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double mag2 {2*par.energy_group/mp}; //cm^2/s^2 + double mag2 {2*particle_info.energy_group/mp}; //cm^2/s^2 - double coef[9]; + double coef2[9]; - coef[0] = -3.271396786375e1; coef[1] = 1.353655609057e1; coef[2] = -5.739328757388; - coef[3] = 1.563154982022; coef[4] = -2.877056004391e-1; coef[5] = 3.482559773737e-2; - coef[6] = -2.631976175590e-3; coef[7] = 1.119543953861e-4; coef[8] = -2.039149852002e-6; + coef2[0] = -3.271396786375e1; coef2[1] = 1.353655609057e1; coef2[2] = -5.739328757388; + coef2[3] = 1.563154982022; coef2[4] = -2.877056004391e-1; coef2[5] = 3.482559773737e-2; + coef2[6] = -2.631976175590e-3; coef2[7] = 1.119543953861e-4; coef2[8] = -2.039149852002e-6; double lnsigma_ion {0}; //cm^2 for (int i=0; i<9; i++) { - lnsigma_ion += coef[i]*std::pow(std::log(fie.electron_temperature),i); + lnsigma_ion += coef2[i]*std::pow(std::log(field_info.electron_temperature),i); } double sigma_ion {exp(lnsigma_ion)/sqrt(mag2)}; @@ -210,20 +212,17 @@ class DG2Physics { double lnalpha {0}; for (int i = 0; i < 9; i++) { for (int j = 0; j < 9; j++) { - lnalpha += coef[i][j]*std::pow(std::log(par.energy_group),i)*std::pow(std::log(fie.ion_temperature),j); + lnalpha += coef[i][j]*std::pow(std::log(particle_info.energy_group),i)*std::pow(std::log(field_info.ion_temperature),j); } } double sigma_cx = std::exp(lnalpha)/sqrt(mag2); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) - auto xx = std::sqrt(-2*std::log(x1))*std::cos(2*std::numbers::pi*x2); - auto yy = std::sqrt(-2*std::log(x1))*std::sin(2*std::numbers::pi*x2); - auto zz = std::sqrt(-2*std::log(y1))*std::cos(2*std::numbers::pi*y2); - auto ux = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(xx))*cos(2*std::numbers::pi*y); - auto uy = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(yy))*sin(2*std::numbers::pi*y); - auto uz = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(zz))*sin(2*std::numbers::pi*y); + auto ux = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(x1))*cos(2*std::numbers::pi*x2); + auto uy = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(x1))*sin(2*std::numbers::pi*x2); + auto uz = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(y1))*sin(2*std::numbers::pi*y2); auto mag_u = std::sqrt(ux*ux + uy*uy + uz*uz); From 16b33bfbc40eda6039e5963f711fc590e57897aa Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 30 Jun 2025 15:58:47 -0400 Subject: [PATCH 009/139] added semicolon --- src/physics/degas2/DG2Physics.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index dec3a65..f6965c1 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -63,7 +63,7 @@ class DG2Physics { KOKKOS_FUNCTION void sample_collision_distance(ParticleInfo &particle_info, const FieldInfo &field_info) const { - using namespace std::numbers + using namespace std::numbers; // example of sampling a random number auto rand_gen = random_pool.get_state(); double x = rand_gen.drand(0., 1.); From 75a3e47f23343896faba913703cd7f4da26b60fc Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 30 Jun 2025 16:06:50 -0400 Subject: [PATCH 010/139] Fixed Pi --- src/physics/degas2/DG2Physics.h | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index f6965c1..d88d8ed 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -73,11 +73,11 @@ class DG2Physics { double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double mag2 {2*particle_info.energy_group/mp}; //cm^2/s^2 - double coef[9]; + double coef2[9]; - coef[0] = -3.271396786375e1; coef[1] = 1.353655609057e1; coef[2] = -5.739328757388; - coef[3] = 1.563154982022; coef[4] = -2.877056004391e-1; coef[5] = 3.482559773737e-2; - coef[6] = -2.631976175590e-3; coef[7] = 1.119543953861e-4; coef[8] = -2.039149852002e-6; + coef2[0] = -3.271396786375e1; coef2[1] = 1.353655609057e1; coef2[2] = -5.739328757388; + coef2[3] = 1.563154982022; coef2[4] = -2.877056004391e-1; coef2[5] = 3.482559773737e-2; + coef2[6] = -2.631976175590e-3; coef2[7] = 1.119543953861e-4; coef2[8] = -2.039149852002e-6; double lnsigma_ion {0}; //cm^2 for (int i=0; i<9; i++) { @@ -144,7 +144,6 @@ class DG2Physics { void collide_particle(ParticleInfo &particle_info, const FieldInfo &field_info) const { - using namespace std::numbers //Generate Random Numbers auto rand_gen = random_pool.get_state(); double x1 = rand_gen.drand(0., 1.); @@ -220,9 +219,9 @@ class DG2Physics { //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) - auto ux = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(x1))*cos(2*std::numbers::pi*x2); - auto uy = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(x1))*sin(2*std::numbers::pi*x2); - auto uz = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(y1))*sin(2*std::numbers::pi*y2); + auto ux = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(x1))*cos(2*M_PI*x2); + auto uy = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(x1))*sin(2*M_PI*x2); + auto uz = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(y1))*sin(2*M_PI*y2); auto mag_u = std::sqrt(ux*ux + uy*uy + uz*uz); From d440b1f258c2f93fb3fa4ca6e844a74421b8a625 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 30 Jun 2025 16:10:18 -0400 Subject: [PATCH 011/139] Fixed std::numbers being in there still --- src/physics/degas2/DG2Physics.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index d88d8ed..0ab39fc 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -63,7 +63,6 @@ class DG2Physics { KOKKOS_FUNCTION void sample_collision_distance(ParticleInfo &particle_info, const FieldInfo &field_info) const { - using namespace std::numbers; // example of sampling a random number auto rand_gen = random_pool.get_state(); double x = rand_gen.drand(0., 1.); @@ -73,7 +72,7 @@ class DG2Physics { double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double mag2 {2*particle_info.energy_group/mp}; //cm^2/s^2 - double coef2[9]; + double coef2 [9]; coef2[0] = -3.271396786375e1; coef2[1] = 1.353655609057e1; coef2[2] = -5.739328757388; coef2[3] = 1.563154982022; coef2[4] = -2.877056004391e-1; coef2[5] = 3.482559773737e-2; From 4a8585da8817c3de97dc5efaa3dd68349b8b39f9 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 30 Jun 2025 16:11:57 -0400 Subject: [PATCH 012/139] Changed coef to coef2 correctly --- src/physics/degas2/DG2Physics.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 0ab39fc..ce04df6 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -80,7 +80,7 @@ class DG2Physics { double lnsigma_ion {0}; //cm^2 for (int i=0; i<9; i++) { - lnsigma_ion += coef[i]*std::pow(std::log(field_info.electron_temperature),i); + lnsigma_ion += coef2[i]*std::pow(std::log(field_info.electron_temperature),i); } double sigma_ion {exp(lnsigma_ion)/sqrt(mag2)}; From bd4b88c7600c1608f5cc49247dd6be13bac0563d Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 30 Jun 2025 16:18:10 -0400 Subject: [PATCH 013/139] Changed std::log to log --- src/physics/degas2/DG2Physics.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index ce04df6..3d6031c 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -80,7 +80,7 @@ class DG2Physics { double lnsigma_ion {0}; //cm^2 for (int i=0; i<9; i++) { - lnsigma_ion += coef2[i]*std::pow(std::log(field_info.electron_temperature),i); + lnsigma_ion += coef2[i]*std::pow(log(field_info.electron_temperature),i); } double sigma_ion {exp(lnsigma_ion)/sqrt(mag2)}; @@ -126,13 +126,13 @@ class DG2Physics { double lnalpha {0}; for (int i = 0; i < 9; i++) { for (int j = 0; j < 9; j++) { - lnalpha += coef[i][j]*std::pow(std::log(particle_info.energy_group),i)*std::pow(std::log(field_info.ion_temperature),j); + lnalpha += coef[i][j]*std::pow(log(particle_info.energy_group),i)*std::pow(log(field_info.ion_temperature),j); } } double sigma_cx = std::exp(lnalpha)/sqrt(mag2); //Generate distance and move particle - double l =-std::log(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 + double l =-log(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 particle_info.position[0] += l*particle_info.direction[0]; particle_info.position[1] += l*particle_info.direction[1]; particle_info.position[2] += l*particle_info.direction[2]; @@ -164,7 +164,7 @@ class DG2Physics { double lnsigma_ion {0}; //cm^2 for (int i=0; i<9; i++) { - lnsigma_ion += coef2[i]*std::pow(std::log(field_info.electron_temperature),i); + lnsigma_ion += coef2[i]*std::pow(log(field_info.electron_temperature),i); } double sigma_ion {exp(lnsigma_ion)/sqrt(mag2)}; @@ -210,7 +210,7 @@ class DG2Physics { double lnalpha {0}; for (int i = 0; i < 9; i++) { for (int j = 0; j < 9; j++) { - lnalpha += coef[i][j]*std::pow(std::log(particle_info.energy_group),i)*std::pow(std::log(field_info.ion_temperature),j); + lnalpha += coef[i][j]*std::pow(log(particle_info.energy_group),i)*std::pow(log(field_info.ion_temperature),j); } } double sigma_cx = std::exp(lnalpha)/sqrt(mag2); From ded643ccdb3cb8929408c746a1a6930d060bc223 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 30 Jun 2025 16:21:20 -0400 Subject: [PATCH 014/139] changed log to logf --- src/physics/degas2/DG2Physics.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 3d6031c..ea77348 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -80,7 +80,7 @@ class DG2Physics { double lnsigma_ion {0}; //cm^2 for (int i=0; i<9; i++) { - lnsigma_ion += coef2[i]*std::pow(log(field_info.electron_temperature),i); + lnsigma_ion += coef2[i]*std::pow(logf(field_info.electron_temperature),i); } double sigma_ion {exp(lnsigma_ion)/sqrt(mag2)}; @@ -126,13 +126,13 @@ class DG2Physics { double lnalpha {0}; for (int i = 0; i < 9; i++) { for (int j = 0; j < 9; j++) { - lnalpha += coef[i][j]*std::pow(log(particle_info.energy_group),i)*std::pow(log(field_info.ion_temperature),j); + lnalpha += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); } } double sigma_cx = std::exp(lnalpha)/sqrt(mag2); //Generate distance and move particle - double l =-log(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 + double l =-logf(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 particle_info.position[0] += l*particle_info.direction[0]; particle_info.position[1] += l*particle_info.direction[1]; particle_info.position[2] += l*particle_info.direction[2]; @@ -164,7 +164,7 @@ class DG2Physics { double lnsigma_ion {0}; //cm^2 for (int i=0; i<9; i++) { - lnsigma_ion += coef2[i]*std::pow(log(field_info.electron_temperature),i); + lnsigma_ion += coef2[i]*std::pow(logf(field_info.electron_temperature),i); } double sigma_ion {exp(lnsigma_ion)/sqrt(mag2)}; @@ -210,7 +210,7 @@ class DG2Physics { double lnalpha {0}; for (int i = 0; i < 9; i++) { for (int j = 0; j < 9; j++) { - lnalpha += coef[i][j]*std::pow(log(particle_info.energy_group),i)*std::pow(log(field_info.ion_temperature),j); + lnalpha += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); } } double sigma_cx = std::exp(lnalpha)/sqrt(mag2); @@ -218,9 +218,9 @@ class DG2Physics { //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) - auto ux = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(x1))*cos(2*M_PI*x2); - auto uy = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(x1))*sin(2*M_PI*x2); - auto uz = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * log(y1))*sin(2*M_PI*y2); + auto ux = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(x1))*cos(2*M_PI*x2); + auto uy = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(x1))*sin(2*M_PI*x2); + auto uz = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(y1))*sin(2*M_PI*y2); auto mag_u = std::sqrt(ux*ux + uy*uy + uz*uz); From d0a6226e5bf5af9c484b6ebe3847ca1c28fe695a Mon Sep 17 00:00:00 2001 From: Gabe Date: Tue, 1 Jul 2025 14:40:12 -0400 Subject: [PATCH 015/139] Add logging and test for both moving and colliding --- test/test_degas2_physics.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 4fa4d02..ba68e8e 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -7,6 +7,7 @@ #include "DG2Physics.h" #include +#include int main(int argc, char *argv[]) { Kokkos::initialize(argc, argv); @@ -43,10 +44,23 @@ int main(int argc, char *argv[]) { }); // Call the function to be tested - Kokkos::parallel_for( + Kokkos::parallel_for( "run physics", numParticles, KOKKOS_LAMBDA(int i) { physics.collide_particle(particles(i), fields(i)); }); + Kokkos::parallel_for( + "run physics", numParticles, KOKKOS_LAMBDA(int i) { + physics.sample_collision_distance(particles(i), fields(i)); + }); + } + while (int n = 0; n Date: Tue, 1 Jul 2025 14:45:02 -0400 Subject: [PATCH 016/139] Fix for loop and endl. --- test/test_degas2_physics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index ba68e8e..4ca0cc9 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -53,14 +53,14 @@ int main(int argc, char *argv[]) { physics.sample_collision_distance(particles(i), fields(i)); }); } - while (int n = 0; n Date: Tue, 1 Jul 2025 14:48:16 -0400 Subject: [PATCH 017/139] Move for loop --- test/test_degas2_physics.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 4ca0cc9..54c6d51 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -52,8 +52,7 @@ int main(int argc, char *argv[]) { "run physics", numParticles, KOKKOS_LAMBDA(int i) { physics.sample_collision_distance(particles(i), fields(i)); }); - } - for (int n = 0; n Date: Tue, 1 Jul 2025 15:02:45 -0400 Subject: [PATCH 018/139] Change to console log --- test/test_degas2_physics.cpp | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 54c6d51..3589691 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -7,7 +7,7 @@ #include "DG2Physics.h" #include -#include + int main(int argc, char *argv[]) { Kokkos::initialize(argc, argv); @@ -53,14 +53,17 @@ int main(int argc, char *argv[]) { physics.sample_collision_distance(particles(i), fields(i)); }); for (int n = 0; n < numParticles; n++) { - std::ofstream outfile("Log.txt"); - - outfile << "Particle #" << n << std::endl; - outfile << "Energy: " << particles(n).energy_group << ", Weight: " << particles(n).weight << std::endl; - outfile << "Position: " << particles(n).position[0] << ", " << particles(n).position[1] << ", " << particles(n).position[2] << std::endl; - outfile << "Direction: " << particles(n).direction[0] << ", " << particles(n).direction[1] << ", " << particles(n).direction[2] << std::endl; - outfile << std::endl; - } + printf("Particle #: %i\n", n); + printf("Energy: %i\n", particles(n).energy_group); + printf("Weight: %f\n", particles(n).weight); + printf("Particle Position: (%f, ", particles(n).position[0]); + printf("Particle Position: %f, ", particles(n).position[1]); + printf("Particle Position: %f)\n", particles(n).position[2]); + printf("Particle Direction: (%f, ", particles(n).direction[0]); + printf("Particle Direction: %f, ", particles(n).direction[1]); + printf("Particle Direction: %f)\n", particles(n).direction[2]); + printf("\n"); + } } Kokkos::finalize(); From 91e485ba7887290289dbbbf9d4500af8d70ab7dc Mon Sep 17 00:00:00 2001 From: Gabe Date: Tue, 1 Jul 2025 15:09:42 -0400 Subject: [PATCH 019/139] Put print in the parallel loop --- test/test_degas2_physics.cpp | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 3589691..9ec7654 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -47,23 +47,19 @@ int main(int argc, char *argv[]) { Kokkos::parallel_for( "run physics", numParticles, KOKKOS_LAMBDA(int i) { physics.collide_particle(particles(i), fields(i)); + physics.sample_collision_distance(particles(i), fields(i)); + + printf("Particle #: %i\n", i); + printf("Energy: %i\n", particles(i).energy_group); + printf("Weight: %f\n", particles(i).weight); + printf("Particle Position: (%f, ", particles(i).position[0]); + printf("Particle Position: %f, ", particles(i).position[1]); + printf("Particle Position: %f)\n", particles(i).position[2]); + printf("Particle Direction: (%f, ", particles(i).direction[0]); + printf("Particle Direction: %f, ", particles(i).direction[1]); + printf("Particle Direction: %f)\n", particles(i).direction[2]); + printf("\n"); }); - Kokkos::parallel_for( - "run physics", numParticles, KOKKOS_LAMBDA(int i) { - physics.sample_collision_distance(particles(i), fields(i)); - }); - for (int n = 0; n < numParticles; n++) { - printf("Particle #: %i\n", n); - printf("Energy: %i\n", particles(n).energy_group); - printf("Weight: %f\n", particles(n).weight); - printf("Particle Position: (%f, ", particles(n).position[0]); - printf("Particle Position: %f, ", particles(n).position[1]); - printf("Particle Position: %f)\n", particles(n).position[2]); - printf("Particle Direction: (%f, ", particles(n).direction[0]); - printf("Particle Direction: %f, ", particles(n).direction[1]); - printf("Particle Direction: %f)\n", particles(n).direction[2]); - printf("\n"); - } } Kokkos::finalize(); From fb5cabdb5826b1ba2c194f84dd1fc48ee55ab733 Mon Sep 17 00:00:00 2001 From: Gabe Date: Tue, 1 Jul 2025 15:35:17 -0400 Subject: [PATCH 020/139] Trying to move data off of gpu --- test/test_degas2_physics.cpp | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 9ec7654..8d51c9d 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -7,6 +7,7 @@ #include "DG2Physics.h" #include +#include int main(int argc, char *argv[]) { @@ -46,22 +47,23 @@ int main(int argc, char *argv[]) { // Call the function to be tested Kokkos::parallel_for( "run physics", numParticles, KOKKOS_LAMBDA(int i) { - physics.collide_particle(particles(i), fields(i)); - physics.sample_collision_distance(particles(i), fields(i)); - - printf("Particle #: %i\n", i); - printf("Energy: %i\n", particles(i).energy_group); - printf("Weight: %f\n", particles(i).weight); - printf("Particle Position: (%f, ", particles(i).position[0]); - printf("Particle Position: %f, ", particles(i).position[1]); - printf("Particle Position: %f)\n", particles(i).position[2]); - printf("Particle Direction: (%f, ", particles(i).direction[0]); - printf("Particle Direction: %f, ", particles(i).direction[1]); - printf("Particle Direction: %f)\n", particles(i).direction[2]); - printf("\n"); + physics.sample_collision_distance(particles(i), fields(i)); + physics.collide_particle(particles(i), fields(i)); }); - } + auto output = create_mirror_view(particles); + + for (int i = 0; i < numParticles; ++i) { + std::ofstream outfile("Log.txt"); + outfile << "Particle #" << i << std::endl; + outfile << "Energy: " << output(i).energy_group << "Weight: " << output(i).weight << std::endl; + outfile << "Position: " << output(i).position[0] << ", " << output(i).position[1] << ", " << output(i).position[2] << std::endl; + outfile << "Direction: " << output(i).direction[0] << ", " << output(i).direction[1] << ", " << output(i).direction[2] << std::endl; + outfile << std::endl; + } + + + } Kokkos::finalize(); return 0; } From f95edd158f3c25dc9e0080705a802aa789795c9c Mon Sep 17 00:00:00 2001 From: Gabe Date: Tue, 1 Jul 2025 15:41:30 -0400 Subject: [PATCH 021/139] trying to copy gpu view onto cpu --- test/test_degas2_physics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 8d51c9d..f0caf6f 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -51,10 +51,10 @@ int main(int argc, char *argv[]) { physics.collide_particle(particles(i), fields(i)); }); auto output = create_mirror_view(particles); + Kokkos::deep_copy(output, particles); + std::ofstream outfile("Log.txt"); for (int i = 0; i < numParticles; ++i) { - std::ofstream outfile("Log.txt"); - outfile << "Particle #" << i << std::endl; outfile << "Energy: " << output(i).energy_group << "Weight: " << output(i).weight << std::endl; outfile << "Position: " << output(i).position[0] << ", " << output(i).position[1] << ", " << output(i).position[2] << std::endl; From 78bca1ef8ebe1289cdf543626f655776e1439769 Mon Sep 17 00:00:00 2001 From: Gabe Date: Tue, 1 Jul 2025 15:48:33 -0400 Subject: [PATCH 022/139] Set more realistic parameters in test --- test/test_degas2_physics.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index f0caf6f..6c0e8fe 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -34,13 +34,13 @@ int main(int argc, char *argv[]) { particles(i).direction[0] = 1; particles(i).direction[1] = 0; particles(i).direction[2] = 0; - particles(i).energy_group = 1; + particles(i).energy_group = 3; particles(i).weight = 1.0; - fields(i).electron_density = 1.0; - fields(i).ion_density = 1.0; - fields(i).electron_temperature = 1.0; - fields(i).ion_temperature = 1.0; + fields(i).electron_density = 1.0e18; + fields(i).ion_density = 1.0e18; + fields(i).electron_temperature = 1000.0; + fields(i).ion_temperature = 1000.0; // and others like this }); From d2cd02540a0af956e0e04d2eea7d3ff2f843d535 Mon Sep 17 00:00:00 2001 From: Gabe Date: Tue, 1 Jul 2025 15:49:06 -0400 Subject: [PATCH 023/139] add space in log --- test/test_degas2_physics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 6c0e8fe..3b5ab68 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -56,7 +56,7 @@ int main(int argc, char *argv[]) { for (int i = 0; i < numParticles; ++i) { outfile << "Particle #" << i << std::endl; - outfile << "Energy: " << output(i).energy_group << "Weight: " << output(i).weight << std::endl; + outfile << "Energy: " << output(i).energy_group << " Weight: " << output(i).weight << std::endl; outfile << "Position: " << output(i).position[0] << ", " << output(i).position[1] << ", " << output(i).position[2] << std::endl; outfile << "Direction: " << output(i).direction[0] << ", " << output(i).direction[1] << ", " << output(i).direction[2] << std::endl; outfile << std::endl; From 116af54f96907a5cf663f3d75f1c1b589587209a Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 2 Jul 2025 09:21:09 -0400 Subject: [PATCH 024/139] Change energy_group type to double --- src/physics/degas2/DG2Physics.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index ea77348..6b4fe28 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -18,7 +18,7 @@ struct ParticleInfo { double position[3]; // Position in space (x, y, z) double direction[3]; // Direction vector (unit vector) double weight; - int energy_group; // Energy group *index* + double energy_group; // Energy group *index* }; struct FieldInfo { From 3a82591c6477c737d3177de27988e82447a9b783 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 2 Jul 2025 09:21:25 -0400 Subject: [PATCH 025/139] Format output as csv --- test/test_degas2_physics.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 3b5ab68..ccc6409 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -31,10 +31,10 @@ int main(int argc, char *argv[]) { particles(i).position[0] = 0; particles(i).position[1] = 0; particles(i).position[2] = 0; - particles(i).direction[0] = 1; + particles(i).direction[0] = 1.0; particles(i).direction[1] = 0; particles(i).direction[2] = 0; - particles(i).energy_group = 3; + particles(i).energy_group = 3.0; particles(i).weight = 1.0; fields(i).electron_density = 1.0e18; @@ -55,11 +55,16 @@ int main(int argc, char *argv[]) { std::ofstream outfile("Log.txt"); for (int i = 0; i < numParticles; ++i) { - outfile << "Particle #" << i << std::endl; - outfile << "Energy: " << output(i).energy_group << " Weight: " << output(i).weight << std::endl; - outfile << "Position: " << output(i).position[0] << ", " << output(i).position[1] << ", " << output(i).position[2] << std::endl; - outfile << "Direction: " << output(i).direction[0] << ", " << output(i).direction[1] << ", " << output(i).direction[2] << std::endl; - outfile << std::endl; + outfile << "Particle #,Energy(eV),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; + outfile << i << ","; + outfile << output(i).energy_group << ","; + outfile << output(i).weight << ","; + outfile << output(i).position[0] << ","; + outfile << output(i).position[1] << ","; + outfile << output(i).position[2] << ","; + outfile << output(i).direction[0] << ","; + outfile << output(i).direction[1] << ","; + outfile << output(i).direction[2] << std::endl; } From bc89e8cdf7058084fe8ebb06bd43988e4b7cf042 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 2 Jul 2025 09:21:25 -0400 Subject: [PATCH 026/139] Format output as csv --- test/test_degas2_physics.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 3b5ab68..7e92f7b 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -31,10 +31,10 @@ int main(int argc, char *argv[]) { particles(i).position[0] = 0; particles(i).position[1] = 0; particles(i).position[2] = 0; - particles(i).direction[0] = 1; + particles(i).direction[0] = 1.0; particles(i).direction[1] = 0; particles(i).direction[2] = 0; - particles(i).energy_group = 3; + particles(i).energy_group = 3.0; particles(i).weight = 1.0; fields(i).electron_density = 1.0e18; @@ -53,13 +53,17 @@ int main(int argc, char *argv[]) { auto output = create_mirror_view(particles); Kokkos::deep_copy(output, particles); std::ofstream outfile("Log.txt"); - + outfile << "Particle #,Energy(eV),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; for (int i = 0; i < numParticles; ++i) { - outfile << "Particle #" << i << std::endl; - outfile << "Energy: " << output(i).energy_group << " Weight: " << output(i).weight << std::endl; - outfile << "Position: " << output(i).position[0] << ", " << output(i).position[1] << ", " << output(i).position[2] << std::endl; - outfile << "Direction: " << output(i).direction[0] << ", " << output(i).direction[1] << ", " << output(i).direction[2] << std::endl; - outfile << std::endl; + outfile << i << ","; + outfile << output(i).energy_group << ","; + outfile << output(i).weight << ","; + outfile << output(i).position[0] << ","; + outfile << output(i).position[1] << ","; + outfile << output(i).position[2] << ","; + outfile << output(i).direction[0] << ","; + outfile << output(i).direction[1] << ","; + outfile << output(i).direction[2] << std::endl; } From 0a2da300f5074787e21becaf466e5212ddfc6eaf Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 2 Jul 2025 09:28:00 -0400 Subject: [PATCH 027/139] Fix CSV formatting --- test/test_degas2_physics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index ccc6409..288de6b 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -53,9 +53,9 @@ int main(int argc, char *argv[]) { auto output = create_mirror_view(particles); Kokkos::deep_copy(output, particles); std::ofstream outfile("Log.txt"); + outfile << "Particle #,Energy(eV),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; for (int i = 0; i < numParticles; ++i) { - outfile << "Particle #,Energy(eV),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; outfile << i << ","; outfile << output(i).energy_group << ","; outfile << output(i).weight << ","; From 7bb9daf0c149f686f7063243ef8beede85b85a69 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 2 Jul 2025 09:40:36 -0400 Subject: [PATCH 028/139] Set particle number to 1000 --- test/test_degas2_physics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 288de6b..8830cdd 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -14,7 +14,7 @@ int main(int argc, char *argv[]) { Kokkos::initialize(argc, argv); { - int numParticles = 10; + int numParticles = 1000; Kokkos::View sigma_t_; // mat, T, g Kokkos::View sigma_a_; // mat, T, g From 1c76c899cc2956f59a66e4d79c54e1fdeb1c8db8 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 2 Jul 2025 13:20:50 -0400 Subject: [PATCH 029/139] Rename variables to be more consistent --- src/physics/degas2/DG2Physics.h | 46 +++++++++++++++++---------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 6b4fe28..0d0b696 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -37,6 +37,7 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); + //cross_sections = Kokkos::View("cross_sections", num_particles); } DG2Physics(const DG2CrossSection cross_section, const int num_particles, @@ -46,6 +47,7 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); + //cross_sections = Kokkos::View("cross_sections", num_particles); } //From Here // sample a new distance and update the particle's position but does not @@ -70,7 +72,7 @@ class DG2Physics { //Compute Ionization Cross Section double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double mag2 {2*particle_info.energy_group/mp}; //cm^2/s^2 + double particle_velocity_squared {2*particle_info.energy_group/mp}; //cm^2/s^2 double coef2 [9]; @@ -78,11 +80,11 @@ class DG2Physics { coef2[3] = 1.563154982022; coef2[4] = -2.877056004391e-1; coef2[5] = 3.482559773737e-2; coef2[6] = -2.631976175590e-3; coef2[7] = 1.119543953861e-4; coef2[8] = -2.039149852002e-6; - double lnsigma_ion {0}; //cm^2 + double lnrate_ion {0}; //cm^2 for (int i=0; i<9; i++) { - lnsigma_ion += coef2[i]*std::pow(logf(field_info.electron_temperature),i); + lnrate_ion += coef2[i]*std::pow(logf(field_info.electron_temperature),i); } - double sigma_ion {exp(lnsigma_ion)/sqrt(mag2)}; + double sigma_ion {exp(lnrate_ion)/sqrt(particle_velocity_squared)}; //Compute Charge Exchange Cross Section double coef [9][9]; //E index, T index @@ -123,13 +125,13 @@ class DG2Physics { coef[8][3] = -2.911233951880e-9; coef[8][4] = 5.117133050290e-9; coef[8][5] = -1.130988250912e-9; coef[8][6] = 1.005189187279e-10; coef[8][7] = -3.989884105603e-12; coef[8][8] = 6.388219930167e-14; - double lnalpha {0}; + double lnrate_cx {0}; for (int i = 0; i < 9; i++) { for (int j = 0; j < 9; j++) { - lnalpha += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); + lnrate_cx += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); } } - double sigma_cx = std::exp(lnalpha)/sqrt(mag2); + double sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); //Generate distance and move particle double l =-logf(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 @@ -154,7 +156,7 @@ class DG2Physics { //Compute Ionization Cross Section double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double mag2 {2*particle_info.energy_group/mp}; //cm^2/s^2 + double particle_velocity_squared {2*particle_info.energy_group/mp}; //cm^2/s^2 double coef2[9]; @@ -162,11 +164,11 @@ class DG2Physics { coef2[3] = 1.563154982022; coef2[4] = -2.877056004391e-1; coef2[5] = 3.482559773737e-2; coef2[6] = -2.631976175590e-3; coef2[7] = 1.119543953861e-4; coef2[8] = -2.039149852002e-6; - double lnsigma_ion {0}; //cm^2 + double lnrate_ion {0}; //cm^2 for (int i=0; i<9; i++) { - lnsigma_ion += coef2[i]*std::pow(logf(field_info.electron_temperature),i); + lnrate_ion += coef2[i]*std::pow(logf(field_info.electron_temperature),i); } - double sigma_ion {exp(lnsigma_ion)/sqrt(mag2)}; + double sigma_ion {exp(lnrate_ion)/sqrt(particle_velocity_squared)}; //Compute Charge Exchange Cross Section double coef [9][9]; //E index, T index @@ -207,28 +209,28 @@ class DG2Physics { coef[8][3] = -2.911233951880e-9; coef[8][4] = 5.117133050290e-9; coef[8][5] = -1.130988250912e-9; coef[8][6] = 1.005189187279e-10; coef[8][7] = -3.989884105603e-12; coef[8][8] = 6.388219930167e-14; - double lnalpha {0}; + double lnrate_cx {0}; for (int i = 0; i < 9; i++) { for (int j = 0; j < 9; j++) { - lnalpha += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); + lnrate_cx += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); } } - double sigma_cx = std::exp(lnalpha)/sqrt(mag2); + double sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) - auto ux = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(x1))*cos(2*M_PI*x2); - auto uy = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(x1))*sin(2*M_PI*x2); - auto uz = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(y1))*sin(2*M_PI*y2); + auto vx = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(x1))*cos(2*M_PI*x2); + auto vy = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(x1))*sin(2*M_PI*x2); + auto vz = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(y1))*sin(2*M_PI*y2); - auto mag_u = std::sqrt(ux*ux + uy*uy + uz*uz); + auto mag_v = std::sqrt(vx*vx + vy*vy + vz*vz); - particle_info.direction[0] = ux/mag_u; - particle_info.direction[1] = uy/mag_u; - particle_info.direction[2] = uz/mag_u; + particle_info.direction[0] = vx/mag_v; + particle_info.direction[1] = vy/mag_v; + particle_info.direction[2] = vz/mag_v; - particle_info.energy_group = 0.5*mp*mag_u*mag_u; + particle_info.energy_group = 0.5*mp*mag_v*mag_v; //Adjust Weights double new_weight = particle_info.weight*(1- From c4c120c8e786caf088692eca2eb40f14eb2f99ed Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 2 Jul 2025 13:21:10 -0400 Subject: [PATCH 030/139] Set density to more reasonable value --- test/test_degas2_physics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 8830cdd..a666bc5 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -37,8 +37,8 @@ int main(int argc, char *argv[]) { particles(i).energy_group = 3.0; particles(i).weight = 1.0; - fields(i).electron_density = 1.0e18; - fields(i).ion_density = 1.0e18; + fields(i).electron_density = 1.0e13; + fields(i).ion_density = 1.0e13; fields(i).electron_temperature = 1000.0; fields(i).ion_temperature = 1000.0; // and others like this From 5466493ba60ae466d22e485f1dc0ddb652c8fc5d Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 09:27:38 -0400 Subject: [PATCH 031/139] Try avoiding calculating the cross sections twice --- src/physics/degas2/DG2Physics.h | 71 +++++---------------------------- 1 file changed, 9 insertions(+), 62 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 0d0b696..a749494 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -37,6 +37,9 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); + sigma_cx = 1.0; + sigma_ion = 1.0; + //cross_sections = Kokkos::View("cross_sections", num_particles); } @@ -47,6 +50,8 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); + igma_cx = 1.0; + sigma_ion = 1.0; //cross_sections = Kokkos::View("cross_sections", num_particles); } //From Here @@ -84,7 +89,7 @@ class DG2Physics { for (int i=0; i<9; i++) { lnrate_ion += coef2[i]*std::pow(logf(field_info.electron_temperature),i); } - double sigma_ion {exp(lnrate_ion)/sqrt(particle_velocity_squared)}; + sigma_ion = exp(lnrate_ion)/sqrt(particle_velocity_squared); //Compute Charge Exchange Cross Section double coef [9][9]; //E index, T index @@ -131,7 +136,7 @@ class DG2Physics { lnrate_cx += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); } } - double sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); + sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); //Generate distance and move particle double l =-logf(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 @@ -156,66 +161,6 @@ class DG2Physics { //Compute Ionization Cross Section double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double particle_velocity_squared {2*particle_info.energy_group/mp}; //cm^2/s^2 - - double coef2[9]; - - coef2[0] = -3.271396786375e1; coef2[1] = 1.353655609057e1; coef2[2] = -5.739328757388; - coef2[3] = 1.563154982022; coef2[4] = -2.877056004391e-1; coef2[5] = 3.482559773737e-2; - coef2[6] = -2.631976175590e-3; coef2[7] = 1.119543953861e-4; coef2[8] = -2.039149852002e-6; - - double lnrate_ion {0}; //cm^2 - for (int i=0; i<9; i++) { - lnrate_ion += coef2[i]*std::pow(logf(field_info.electron_temperature),i); - } - double sigma_ion {exp(lnrate_ion)/sqrt(particle_velocity_squared)}; - - //Compute Charge Exchange Cross Section - double coef [9][9]; //E index, T index - - coef[0][0] = -1.829079581680e1; coef[0][1] = 2.169137615703e-1; coef[0][2] = 4.307131243894e-2; - coef[0][3] = -5.754895093075e-4; coef[0][4] = -1.552077120204e-3; coef[0][5] = -1.876800283030e-4; - coef[0][6] = 1.125490270962e-4; coef[0][7] = -1.238982763007e-5; coef[0][8] = 4.163596197181e-07; - - coef[1][0] = 1.640252721210e-1; coef[1][1] = -1.106722014459e-1; coef[1][2] = 8.948693624917e-3; - coef[1][3] = 6.062141761233e-3; coef[1][4] = -1.210431587568e-3; coef[1][5] = -4.052878751584e-5; - coef[1][6] = 2.875900435895e-5; coef[1][7] = -2.616998139678e-6; coef[1][8] = 7.558092849125e-8; - - coef[2][0] = 3.364564509137e-2; coef[2][1] = -1.382158680424e-3; coef[2][2] = -1.209480567154e-02; - coef[2][3] = 1.075907881928e-3; coef[2][4] = 8.297212635856e-4; coef[2][5] = -1.907025662962e-04; - coef[2][6] = 1.338839628570e-5; coef[2][7] = -1.171762874107e-7; coef[2][8] = -1.328404104165e-8; - - coef[3][0] = 9.530225559189e-3; coef[3][1] = 7.348786286628e-3; coef[3][2] = -3.675019470470e-4; - coef[3][3] = -8.119301728339e-4; coef[3][4] = 1.361661816974e-4; coef[3][5] = 1.141663041636e-5; - coef[3][6] = -4.340802793033e-6; coef[3][7] = 3.517971869029e-7; coef[3][8] = -9.170850253981e-09; - - coef[4][0] = -8.519413589968e-4; coef[4][1] = -6.343059502294e-4; coef[4][2] = 1.039643390686e-3; - coef[4][3] = 8.911036876068e-6; coef[4][4] = -1.008928628425e-4; coef[4][5] = 1.775681984457e-05; - coef[4][6] = -7.003521917385e-7; coef[4][7] = -4.928692832866e-8; coef[4][8] = 3.208853883734e-9; - - coef[5][0] = -1.247583860943e-3; coef[5][1] = -1.919569450380e-4; coef[5][2] = -1.553840717902e-4; - coef[5][3] = 3.175388949811e-5; coef[5][4] = 1.080693990468e-5; coef[5][5] = -3.149286923815e-6; - coef[5][6] = 2.318308730487e-7; coef[5][7] = 1.756388998863e-10; coef[5][8] = -3.952740758950e-10; - - coef[6][0] = 3.014307545716e-4; coef[6][1] = 4.075019351738e-5; coef[6][2] = 2.670827249272e-6; - coef[6][3] = -4.515123641755e-6; coef[6][4] = 5.106059413591e-7; coef[6][5] = 3.105491554749e-8; - coef[6][6] = -6.030983538280e-9; coef[6][7] = -1.446756795654e-10; coef[6][8] = 2.739558475782e-11; - - coef[7][0] = -2.499323170044e-5; coef[7][1] = -2.850044983009e-6; coef[7][2] = 7.695300597935e-7; - coef[7][3] = 2.187439283954e-7; coef[7][4] = -1.299275586093e-7; coef[7][5] = 2.274394089017e-8; - coef[7][6] = -1.755944926274e-9; coef[7][7] = 7.143183138281e-11; coef[7][8] = -1.693040208927e-12; - - coef[8][0] = 6.932627237765e-7; coef[8][1] = 6.966822400446e-8; coef[8][2] = -3.783302281524e-8; - coef[8][3] = -2.911233951880e-9; coef[8][4] = 5.117133050290e-9; coef[8][5] = -1.130988250912e-9; - coef[8][6] = 1.005189187279e-10; coef[8][7] = -3.989884105603e-12; coef[8][8] = 6.388219930167e-14; - - double lnrate_cx {0}; - for (int i = 0; i < 9; i++) { - for (int j = 0; j < 9; j++) { - lnrate_cx += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); - } - } - double sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) @@ -256,5 +201,7 @@ class DG2Physics { DG2CrossSection cross_section; Kokkos::View particle_velocities; + double sigma_cx; + double sigma_ion; }; #endif // PUMITALLYOPENMC_DG2PHYSICS_H From ebfc1ebc14618cd55510d69bb88a028a109658a9 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 09:27:38 -0400 Subject: [PATCH 032/139] Try avoiding calculating the cross sections twice --- src/physics/degas2/DG2Physics.h | 71 +++++---------------------------- 1 file changed, 9 insertions(+), 62 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 0d0b696..a749494 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -37,6 +37,9 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); + sigma_cx = 1.0; + sigma_ion = 1.0; + //cross_sections = Kokkos::View("cross_sections", num_particles); } @@ -47,6 +50,8 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); + igma_cx = 1.0; + sigma_ion = 1.0; //cross_sections = Kokkos::View("cross_sections", num_particles); } //From Here @@ -84,7 +89,7 @@ class DG2Physics { for (int i=0; i<9; i++) { lnrate_ion += coef2[i]*std::pow(logf(field_info.electron_temperature),i); } - double sigma_ion {exp(lnrate_ion)/sqrt(particle_velocity_squared)}; + sigma_ion = exp(lnrate_ion)/sqrt(particle_velocity_squared); //Compute Charge Exchange Cross Section double coef [9][9]; //E index, T index @@ -131,7 +136,7 @@ class DG2Physics { lnrate_cx += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); } } - double sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); + sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); //Generate distance and move particle double l =-logf(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 @@ -156,66 +161,6 @@ class DG2Physics { //Compute Ionization Cross Section double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double particle_velocity_squared {2*particle_info.energy_group/mp}; //cm^2/s^2 - - double coef2[9]; - - coef2[0] = -3.271396786375e1; coef2[1] = 1.353655609057e1; coef2[2] = -5.739328757388; - coef2[3] = 1.563154982022; coef2[4] = -2.877056004391e-1; coef2[5] = 3.482559773737e-2; - coef2[6] = -2.631976175590e-3; coef2[7] = 1.119543953861e-4; coef2[8] = -2.039149852002e-6; - - double lnrate_ion {0}; //cm^2 - for (int i=0; i<9; i++) { - lnrate_ion += coef2[i]*std::pow(logf(field_info.electron_temperature),i); - } - double sigma_ion {exp(lnrate_ion)/sqrt(particle_velocity_squared)}; - - //Compute Charge Exchange Cross Section - double coef [9][9]; //E index, T index - - coef[0][0] = -1.829079581680e1; coef[0][1] = 2.169137615703e-1; coef[0][2] = 4.307131243894e-2; - coef[0][3] = -5.754895093075e-4; coef[0][4] = -1.552077120204e-3; coef[0][5] = -1.876800283030e-4; - coef[0][6] = 1.125490270962e-4; coef[0][7] = -1.238982763007e-5; coef[0][8] = 4.163596197181e-07; - - coef[1][0] = 1.640252721210e-1; coef[1][1] = -1.106722014459e-1; coef[1][2] = 8.948693624917e-3; - coef[1][3] = 6.062141761233e-3; coef[1][4] = -1.210431587568e-3; coef[1][5] = -4.052878751584e-5; - coef[1][6] = 2.875900435895e-5; coef[1][7] = -2.616998139678e-6; coef[1][8] = 7.558092849125e-8; - - coef[2][0] = 3.364564509137e-2; coef[2][1] = -1.382158680424e-3; coef[2][2] = -1.209480567154e-02; - coef[2][3] = 1.075907881928e-3; coef[2][4] = 8.297212635856e-4; coef[2][5] = -1.907025662962e-04; - coef[2][6] = 1.338839628570e-5; coef[2][7] = -1.171762874107e-7; coef[2][8] = -1.328404104165e-8; - - coef[3][0] = 9.530225559189e-3; coef[3][1] = 7.348786286628e-3; coef[3][2] = -3.675019470470e-4; - coef[3][3] = -8.119301728339e-4; coef[3][4] = 1.361661816974e-4; coef[3][5] = 1.141663041636e-5; - coef[3][6] = -4.340802793033e-6; coef[3][7] = 3.517971869029e-7; coef[3][8] = -9.170850253981e-09; - - coef[4][0] = -8.519413589968e-4; coef[4][1] = -6.343059502294e-4; coef[4][2] = 1.039643390686e-3; - coef[4][3] = 8.911036876068e-6; coef[4][4] = -1.008928628425e-4; coef[4][5] = 1.775681984457e-05; - coef[4][6] = -7.003521917385e-7; coef[4][7] = -4.928692832866e-8; coef[4][8] = 3.208853883734e-9; - - coef[5][0] = -1.247583860943e-3; coef[5][1] = -1.919569450380e-4; coef[5][2] = -1.553840717902e-4; - coef[5][3] = 3.175388949811e-5; coef[5][4] = 1.080693990468e-5; coef[5][5] = -3.149286923815e-6; - coef[5][6] = 2.318308730487e-7; coef[5][7] = 1.756388998863e-10; coef[5][8] = -3.952740758950e-10; - - coef[6][0] = 3.014307545716e-4; coef[6][1] = 4.075019351738e-5; coef[6][2] = 2.670827249272e-6; - coef[6][3] = -4.515123641755e-6; coef[6][4] = 5.106059413591e-7; coef[6][5] = 3.105491554749e-8; - coef[6][6] = -6.030983538280e-9; coef[6][7] = -1.446756795654e-10; coef[6][8] = 2.739558475782e-11; - - coef[7][0] = -2.499323170044e-5; coef[7][1] = -2.850044983009e-6; coef[7][2] = 7.695300597935e-7; - coef[7][3] = 2.187439283954e-7; coef[7][4] = -1.299275586093e-7; coef[7][5] = 2.274394089017e-8; - coef[7][6] = -1.755944926274e-9; coef[7][7] = 7.143183138281e-11; coef[7][8] = -1.693040208927e-12; - - coef[8][0] = 6.932627237765e-7; coef[8][1] = 6.966822400446e-8; coef[8][2] = -3.783302281524e-8; - coef[8][3] = -2.911233951880e-9; coef[8][4] = 5.117133050290e-9; coef[8][5] = -1.130988250912e-9; - coef[8][6] = 1.005189187279e-10; coef[8][7] = -3.989884105603e-12; coef[8][8] = 6.388219930167e-14; - - double lnrate_cx {0}; - for (int i = 0; i < 9; i++) { - for (int j = 0; j < 9; j++) { - lnrate_cx += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); - } - } - double sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) @@ -256,5 +201,7 @@ class DG2Physics { DG2CrossSection cross_section; Kokkos::View particle_velocities; + double sigma_cx; + double sigma_ion; }; #endif // PUMITALLYOPENMC_DG2PHYSICS_H From 7e46ec197b4064c8c4e247ba900b642da6a8b6fc Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 09:30:56 -0400 Subject: [PATCH 033/139] Fix type --- src/physics/degas2/DG2Physics.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index a749494..a0ae339 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -50,7 +50,7 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); - igma_cx = 1.0; + sigma_cx = 1.0; sigma_ion = 1.0; //cross_sections = Kokkos::View("cross_sections", num_particles); } From 62873bf019e8777b782c6453cc82a438c67e27ae Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 09:33:47 -0400 Subject: [PATCH 034/139] Fix typo --- src/physics/degas2/DG2Physics.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 179c335..4224abb 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -37,8 +37,8 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); - sigma_cx = 1.0; - sigma_ion = 1.0; + double sigma_cx = 1.0; + double sigma_ion = 1.0; //cross_sections = Kokkos::View("cross_sections", num_particles); } @@ -50,8 +50,8 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); - sigma_cx = 1.0; - sigma_ion = 1.0; + double sigma_cx = 1.0; + double sigma_ion = 1.0; //cross_sections = Kokkos::View("cross_sections", num_particles); } //From Here @@ -136,7 +136,7 @@ class DG2Physics { lnrate_cx += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); } } - double sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); + sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); //Generate distance and move particle double l =-logf(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 From d57122cc89583da4b642eaa0ceabced438696207 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 09:34:51 -0400 Subject: [PATCH 035/139] Fix typo --- src/physics/degas2/DG2Physics.h | 60 --------------------------------- 1 file changed, 60 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 4224abb..258ee6f 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -161,66 +161,6 @@ class DG2Physics { //Compute Ionization Cross Section double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double particle_velocity_squared {2*particle_info.energy_group/mp}; //cm^2/s^2 - - double coef2[9]; - - coef2[0] = -3.271396786375e1; coef2[1] = 1.353655609057e1; coef2[2] = -5.739328757388; - coef2[3] = 1.563154982022; coef2[4] = -2.877056004391e-1; coef2[5] = 3.482559773737e-2; - coef2[6] = -2.631976175590e-3; coef2[7] = 1.119543953861e-4; coef2[8] = -2.039149852002e-6; - - double lnrate_ion {0}; //cm^2 - for (int i=0; i<9; i++) { - lnrate_ion += coef2[i]*std::pow(logf(field_info.electron_temperature),i); - } - double sigma_ion {exp(lnrate_ion)/sqrt(particle_velocity_squared)}; - - //Compute Charge Exchange Cross Section - double coef [9][9]; //E index, T index - - coef[0][0] = -1.829079581680e1; coef[0][1] = 2.169137615703e-1; coef[0][2] = 4.307131243894e-2; - coef[0][3] = -5.754895093075e-4; coef[0][4] = -1.552077120204e-3; coef[0][5] = -1.876800283030e-4; - coef[0][6] = 1.125490270962e-4; coef[0][7] = -1.238982763007e-5; coef[0][8] = 4.163596197181e-07; - - coef[1][0] = 1.640252721210e-1; coef[1][1] = -1.106722014459e-1; coef[1][2] = 8.948693624917e-3; - coef[1][3] = 6.062141761233e-3; coef[1][4] = -1.210431587568e-3; coef[1][5] = -4.052878751584e-5; - coef[1][6] = 2.875900435895e-5; coef[1][7] = -2.616998139678e-6; coef[1][8] = 7.558092849125e-8; - - coef[2][0] = 3.364564509137e-2; coef[2][1] = -1.382158680424e-3; coef[2][2] = -1.209480567154e-02; - coef[2][3] = 1.075907881928e-3; coef[2][4] = 8.297212635856e-4; coef[2][5] = -1.907025662962e-04; - coef[2][6] = 1.338839628570e-5; coef[2][7] = -1.171762874107e-7; coef[2][8] = -1.328404104165e-8; - - coef[3][0] = 9.530225559189e-3; coef[3][1] = 7.348786286628e-3; coef[3][2] = -3.675019470470e-4; - coef[3][3] = -8.119301728339e-4; coef[3][4] = 1.361661816974e-4; coef[3][5] = 1.141663041636e-5; - coef[3][6] = -4.340802793033e-6; coef[3][7] = 3.517971869029e-7; coef[3][8] = -9.170850253981e-09; - - coef[4][0] = -8.519413589968e-4; coef[4][1] = -6.343059502294e-4; coef[4][2] = 1.039643390686e-3; - coef[4][3] = 8.911036876068e-6; coef[4][4] = -1.008928628425e-4; coef[4][5] = 1.775681984457e-05; - coef[4][6] = -7.003521917385e-7; coef[4][7] = -4.928692832866e-8; coef[4][8] = 3.208853883734e-9; - - coef[5][0] = -1.247583860943e-3; coef[5][1] = -1.919569450380e-4; coef[5][2] = -1.553840717902e-4; - coef[5][3] = 3.175388949811e-5; coef[5][4] = 1.080693990468e-5; coef[5][5] = -3.149286923815e-6; - coef[5][6] = 2.318308730487e-7; coef[5][7] = 1.756388998863e-10; coef[5][8] = -3.952740758950e-10; - - coef[6][0] = 3.014307545716e-4; coef[6][1] = 4.075019351738e-5; coef[6][2] = 2.670827249272e-6; - coef[6][3] = -4.515123641755e-6; coef[6][4] = 5.106059413591e-7; coef[6][5] = 3.105491554749e-8; - coef[6][6] = -6.030983538280e-9; coef[6][7] = -1.446756795654e-10; coef[6][8] = 2.739558475782e-11; - - coef[7][0] = -2.499323170044e-5; coef[7][1] = -2.850044983009e-6; coef[7][2] = 7.695300597935e-7; - coef[7][3] = 2.187439283954e-7; coef[7][4] = -1.299275586093e-7; coef[7][5] = 2.274394089017e-8; - coef[7][6] = -1.755944926274e-9; coef[7][7] = 7.143183138281e-11; coef[7][8] = -1.693040208927e-12; - - coef[8][0] = 6.932627237765e-7; coef[8][1] = 6.966822400446e-8; coef[8][2] = -3.783302281524e-8; - coef[8][3] = -2.911233951880e-9; coef[8][4] = 5.117133050290e-9; coef[8][5] = -1.130988250912e-9; - coef[8][6] = 1.005189187279e-10; coef[8][7] = -3.989884105603e-12; coef[8][8] = 6.388219930167e-14; - - double lnrate_cx {0}; - for (int i = 0; i < 9; i++) { - for (int j = 0; j < 9; j++) { - lnrate_cx += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); - } - } - double sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) From 3280456b7213e53296b2fd3fe5b88a01bd85e6e1 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 09:36:37 -0400 Subject: [PATCH 036/139] Fix typo --- src/physics/degas2/DG2Physics.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 258ee6f..7703a8e 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -201,5 +201,7 @@ class DG2Physics { DG2CrossSection cross_section; Kokkos::View particle_velocities; + double sigma_cx; + double sigma_ion; }; #endif // PUMITALLYOPENMC_DG2PHYSICS_H From a511e81c6162466fd894e1f1fa758ac58f8631df Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 09:37:58 -0400 Subject: [PATCH 037/139] Try putting declaration earlier --- src/physics/degas2/DG2Physics.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 7703a8e..006dcad 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -66,6 +66,8 @@ class DG2Physics { } */ + double sigma_cx; + double sigma_ion; KOKKOS_FUNCTION void sample_collision_distance(ParticleInfo &particle_info, @@ -201,7 +203,6 @@ class DG2Physics { DG2CrossSection cross_section; Kokkos::View particle_velocities; - double sigma_cx; - double sigma_ion; + }; #endif // PUMITALLYOPENMC_DG2PHYSICS_H From 4657a90fb916cfe442b8420223871e172012eebd Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 09:39:57 -0400 Subject: [PATCH 038/139] Try fixing declaration --- src/physics/degas2/DG2Physics.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 006dcad..e675ceb 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -37,8 +37,8 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); - double sigma_cx = 1.0; - double sigma_ion = 1.0; + //public double sigma_cx = 1.0; + //public double sigma_ion = 1.0; //cross_sections = Kokkos::View("cross_sections", num_particles); } @@ -50,8 +50,8 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); - double sigma_cx = 1.0; - double sigma_ion = 1.0; + //public double sigma_cx = 1.0; + //public double sigma_ion = 1.0; //cross_sections = Kokkos::View("cross_sections", num_particles); } //From Here @@ -66,8 +66,8 @@ class DG2Physics { } */ - double sigma_cx; - double sigma_ion; + public double sigma_cx; + public double sigma_ion; KOKKOS_FUNCTION void sample_collision_distance(ParticleInfo &particle_info, From 76f2890793e680ec137619c6edea8ba6f28105a5 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 09:45:45 -0400 Subject: [PATCH 039/139] Clear some non-functional code (note will not compile) --- src/physics/degas2/DG2Physics.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index e675ceb..d9551f0 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -37,9 +37,6 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); - //public double sigma_cx = 1.0; - //public double sigma_ion = 1.0; - //cross_sections = Kokkos::View("cross_sections", num_particles); } @@ -50,8 +47,6 @@ class DG2Physics { // Initialize particle velocities particle_velocities = Kokkos::View("particle_velocities", num_particles); - //public double sigma_cx = 1.0; - //public double sigma_ion = 1.0; //cross_sections = Kokkos::View("cross_sections", num_particles); } //From Here @@ -66,8 +61,6 @@ class DG2Physics { } */ - public double sigma_cx; - public double sigma_ion; KOKKOS_FUNCTION void sample_collision_distance(ParticleInfo &particle_info, From 2cbf18371c965588b3c9e0aebf3ea19297226956 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 14:33:51 -0400 Subject: [PATCH 040/139] Store energy in a kokkos::view inside of DG2Physics.h and add index to ParticleInfo struct. --- src/physics/degas2/DG2Physics.h | 75 ++++++++++++++++++--------------- test/test_degas2_physics.cpp | 1 + 2 files changed, 42 insertions(+), 34 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index d9551f0..58e16a2 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -19,6 +19,7 @@ struct ParticleInfo { double direction[3]; // Direction vector (unit vector) double weight; double energy_group; // Energy group *index* + int particle_index; }; struct FieldInfo { @@ -35,8 +36,8 @@ class DG2Physics { const int seed = SEED) : random_pool(seed), cross_section(cross_section_file) { // Initialize particle velocities - particle_velocities = - Kokkos::View("particle_velocities", num_particles); + particle_energy = + Kokkos::View("particle_energy", num_particles); //cross_sections = Kokkos::View("cross_sections", num_particles); } @@ -45,35 +46,17 @@ class DG2Physics { : random_pool(seed), cross_section(cross_section) { // Initialize particle velocities - particle_velocities = - Kokkos::View("particle_velocities", num_particles); + particle_energy = + Kokkos::View("particle_energy", num_particles); //cross_sections = Kokkos::View("cross_sections", num_particles); } - //From Here - // sample a new distance and update the particle's position but does not - // change direction - /* - //Functions to evaluate the cross sections. Need to talk to Fuad - //For now I'll put them in the functions - KOKKOS_FUNCTION - double ionization_cross_section(ParticleInfo &particle_info, - const FieldInfo &field_info){ - - } - */ KOKKOS_FUNCTION - void sample_collision_distance(ParticleInfo &particle_info, - const FieldInfo &field_info) const { - // example of sampling a random number - auto rand_gen = random_pool.get_state(); - double x = rand_gen.drand(0., 1.); - random_pool.free_state(rand_gen); + double ionization_cross_section(const double energy, const double e_temperature){ + double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 + double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 //Compute Ionization Cross Section - double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double particle_velocity_squared {2*particle_info.energy_group/mp}; //cm^2/s^2 - double coef2 [9]; coef2[0] = -3.271396786375e1; coef2[1] = 1.353655609057e1; coef2[2] = -5.739328757388; @@ -82,11 +65,17 @@ class DG2Physics { double lnrate_ion {0}; //cm^2 for (int i=0; i<9; i++) { - lnrate_ion += coef2[i]*std::pow(logf(field_info.electron_temperature),i); + lnrate_ion += coef2[i]*std::pow(logf(e_temperature),i); } - sigma_ion = exp(lnrate_ion)/sqrt(particle_velocity_squared); + return exp(lnrate_ion)/sqrt(particle_velocity_squared); + } - //Compute Charge Exchange Cross Section + KOKKOS_FUNCTION + double charge_exchange_cross_section(const double energy, const double ion_temperature){ + double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 + double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 + + //Compute Charge Exchange Cross Section double coef [9][9]; //E index, T index coef[0][0] = -1.829079581680e1; coef[0][1] = 2.169137615703e-1; coef[0][2] = 4.307131243894e-2; @@ -128,10 +117,25 @@ class DG2Physics { double lnrate_cx {0}; for (int i = 0; i < 9; i++) { for (int j = 0; j < 9; j++) { - lnrate_cx += coef[i][j]*std::pow(logf(particle_info.energy_group),i)*std::pow(logf(field_info.ion_temperature),j); + lnrate_cx += coef[i][j]*std::pow(logf(energy),i)*std::pow(logf(ion_temperature),j); } } - sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); + return std::exp(lnrate_cx)/sqrt(particle_velocity_squared); + } + + + KOKKOS_FUNCTION + void sample_collision_distance(ParticleInfo &particle_info, + const FieldInfo &field_info) const { + // example of sampling a random number + auto rand_gen = random_pool.get_state(); + double x = rand_gen.drand(0., 1.); + random_pool.free_state(rand_gen); + + double energy = particle_energy(particle_info.particle_index); + + double sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); + double sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); //Generate distance and move particle double l =-logf(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 @@ -154,11 +158,14 @@ class DG2Physics { double ww = rand_gen.drand(0., 1.); random_pool.free_state(rand_gen); - //Compute Ionization Cross Section - double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 + double energy = particle_energy(particle_info.particle_index); + + double sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); + double sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) + double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 auto vx = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(x1))*cos(2*M_PI*x2); auto vy = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(x1))*sin(2*M_PI*x2); @@ -170,7 +177,7 @@ class DG2Physics { particle_info.direction[1] = vy/mag_v; particle_info.direction[2] = vz/mag_v; - particle_info.energy_group = 0.5*mp*mag_v*mag_v; + particle_energy(particle_info.particle_index) = 0.5*mp*mag_v*mag_v; //Adjust Weights double new_weight = particle_info.weight*(1- @@ -195,7 +202,7 @@ class DG2Physics { random_pool_t random_pool; DG2CrossSection cross_section; - Kokkos::View particle_velocities; + Kokkos::View particle_energy; }; #endif // PUMITALLYOPENMC_DG2PHYSICS_H diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index a666bc5..1e06b39 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -36,6 +36,7 @@ int main(int argc, char *argv[]) { particles(i).direction[2] = 0; particles(i).energy_group = 3.0; particles(i).weight = 1.0; + particles(i).particle_index = i; fields(i).electron_density = 1.0e13; fields(i).ion_density = 1.0e13; From eda9e6a7e0d6a3e4252523491daa3814e5ff10bf Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 14:41:20 -0400 Subject: [PATCH 041/139] remove const in function definition --- src/physics/degas2/DG2Physics.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 58e16a2..9fc73f6 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -52,7 +52,7 @@ class DG2Physics { } KOKKOS_FUNCTION - double ionization_cross_section(const double energy, const double e_temperature){ + double ionization_cross_section(double energy, double e_temperature){ double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 @@ -71,7 +71,7 @@ class DG2Physics { } KOKKOS_FUNCTION - double charge_exchange_cross_section(const double energy, const double ion_temperature){ + double charge_exchange_cross_section(double energy, double ion_temperature){ double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 From 47b85076e797c6f9927b36bfa767ed81cdc3a119 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 14:53:22 -0400 Subject: [PATCH 042/139] change sigmas to auto --- src/physics/degas2/DG2Physics.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 9fc73f6..22dd3b7 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -52,7 +52,7 @@ class DG2Physics { } KOKKOS_FUNCTION - double ionization_cross_section(double energy, double e_temperature){ + double ionization_cross_section(const double energy, const double e_temperature){ double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 @@ -71,7 +71,7 @@ class DG2Physics { } KOKKOS_FUNCTION - double charge_exchange_cross_section(double energy, double ion_temperature){ + double charge_exchange_cross_section(const double energy, const double ion_temperature){ double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 @@ -134,8 +134,8 @@ class DG2Physics { double energy = particle_energy(particle_info.particle_index); - double sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); - double sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); + auto sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); + auto sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); //Generate distance and move particle double l =-logf(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 @@ -160,8 +160,8 @@ class DG2Physics { double energy = particle_energy(particle_info.particle_index); - double sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); - double sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); + auto sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); + auto sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) From 0aaf1d0a74eeb59540eef4116eec2f52c0a1d061 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 15:04:24 -0400 Subject: [PATCH 043/139] explicitly return a variable previously declared as double --- src/physics/degas2/DG2Physics.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 22dd3b7..e42f2aa 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -67,7 +67,8 @@ class DG2Physics { for (int i=0; i<9; i++) { lnrate_ion += coef2[i]*std::pow(logf(e_temperature),i); } - return exp(lnrate_ion)/sqrt(particle_velocity_squared); + double sigma_ion = exp(lnrate_ion)/sqrt(particle_velocity_squared); + return sigma_ion; } KOKKOS_FUNCTION @@ -120,7 +121,8 @@ class DG2Physics { lnrate_cx += coef[i][j]*std::pow(logf(energy),i)*std::pow(logf(ion_temperature),j); } } - return std::exp(lnrate_cx)/sqrt(particle_velocity_squared); + double sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); + return sigma_cx; } From 663cd96de95db07c06aba7e6336516db042cebdf Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 15:06:32 -0400 Subject: [PATCH 044/139] Explicitly call DG2Physics.foo --- src/physics/degas2/DG2Physics.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index e42f2aa..483efcc 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -136,8 +136,8 @@ class DG2Physics { double energy = particle_energy(particle_info.particle_index); - auto sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); - auto sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); + auto sigma_ion = DG2Physics.ionization_cross_section(energy, field_info.electron_temperature); + auto sigma_cx = DG2Physics.charge_exchange_cross_section(energy, field_info.ion_temperature); //Generate distance and move particle double l =-logf(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 @@ -162,8 +162,8 @@ class DG2Physics { double energy = particle_energy(particle_info.particle_index); - auto sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); - auto sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); + auto sigma_ion = DG2Physics.ionization_cross_section(energy, field_info.electron_temperature); + auto sigma_cx = DG2Physics.charge_exchange_cross_section(energy, field_info.ion_temperature); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) From 35f744452587315e5e2f78329d3867bd8d4821c0 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 15:08:18 -0400 Subject: [PATCH 045/139] Try calling with :: instead --- src/physics/degas2/DG2Physics.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 483efcc..f3d75d1 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -162,8 +162,8 @@ class DG2Physics { double energy = particle_energy(particle_info.particle_index); - auto sigma_ion = DG2Physics.ionization_cross_section(energy, field_info.electron_temperature); - auto sigma_cx = DG2Physics.charge_exchange_cross_section(energy, field_info.ion_temperature); + auto sigma_ion = DG2Physics::ionization_cross_section(energy, field_info.electron_temperature); + auto sigma_cx = DG2Physics::charge_exchange_cross_section(energy, field_info.ion_temperature); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) From 9bd33fd84662cea0b84f9a19c06d8238d912b3ba Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 15:23:23 -0400 Subject: [PATCH 046/139] Try making sigma a const? --- src/physics/degas2/DG2Physics.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index f3d75d1..fce6a52 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -136,8 +136,8 @@ class DG2Physics { double energy = particle_energy(particle_info.particle_index); - auto sigma_ion = DG2Physics.ionization_cross_section(energy, field_info.electron_temperature); - auto sigma_cx = DG2Physics.charge_exchange_cross_section(energy, field_info.ion_temperature); + const auto sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); + const auto sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); //Generate distance and move particle double l =-logf(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 @@ -162,8 +162,8 @@ class DG2Physics { double energy = particle_energy(particle_info.particle_index); - auto sigma_ion = DG2Physics::ionization_cross_section(energy, field_info.electron_temperature); - auto sigma_cx = DG2Physics::charge_exchange_cross_section(energy, field_info.ion_temperature); + const auto sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); + const auto sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) From cdde6b32b86121a9d21f3013b19a860176e625f8 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 15:27:46 -0400 Subject: [PATCH 047/139] How about making the functions constant ooh --- src/physics/degas2/DG2Physics.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index fce6a52..8e14164 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -52,7 +52,7 @@ class DG2Physics { } KOKKOS_FUNCTION - double ionization_cross_section(const double energy, const double e_temperature){ + const double ionization_cross_section(const double energy, const double e_temperature){ double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 @@ -72,7 +72,7 @@ class DG2Physics { } KOKKOS_FUNCTION - double charge_exchange_cross_section(const double energy, const double ion_temperature){ + const double charge_exchange_cross_section(const double energy, const double ion_temperature){ double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 @@ -136,8 +136,8 @@ class DG2Physics { double energy = particle_energy(particle_info.particle_index); - const auto sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); - const auto sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); + auto sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); + auto sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); //Generate distance and move particle double l =-logf(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 @@ -162,8 +162,8 @@ class DG2Physics { double energy = particle_energy(particle_info.particle_index); - const auto sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); - const auto sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); + auto sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); + auto sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) From f393060899bbb0a322138287b7caa82b010df8ce Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 15:30:23 -0400 Subject: [PATCH 048/139] How about making the functions constant ooh --- src/physics/degas2/DG2Physics.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 8e14164..e8f6861 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -52,7 +52,7 @@ class DG2Physics { } KOKKOS_FUNCTION - const double ionization_cross_section(const double energy, const double e_temperature){ + double ionization_cross_section(const double energy, const double e_temperature) const{ double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 @@ -72,7 +72,7 @@ class DG2Physics { } KOKKOS_FUNCTION - const double charge_exchange_cross_section(const double energy, const double ion_temperature){ + double charge_exchange_cross_section(const double energy, const double ion_temperature) const{ double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 From bb1ce98a6275cbdb4fa610f780e0a40ee3a12f24 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 15:41:08 -0400 Subject: [PATCH 049/139] Changing the logging to debug --- src/physics/degas2/DG2Physics.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index e8f6861..457ca84 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -35,7 +35,7 @@ class DG2Physics { DG2Physics(const std::string &cross_section_file, const int num_particles, const int seed = SEED) : random_pool(seed), cross_section(cross_section_file) { - // Initialize particle velocities + // Initialize particle energy particle_energy = Kokkos::View("particle_energy", num_particles); //cross_sections = Kokkos::View("cross_sections", num_particles); @@ -45,7 +45,7 @@ class DG2Physics { const int seed = SEED) : random_pool(seed), cross_section(cross_section) { - // Initialize particle velocities + // Initialize particle energy particle_energy = Kokkos::View("particle_energy", num_particles); //cross_sections = Kokkos::View("cross_sections", num_particles); @@ -180,7 +180,7 @@ class DG2Physics { particle_info.direction[2] = vz/mag_v; particle_energy(particle_info.particle_index) = 0.5*mp*mag_v*mag_v; - + particle_info.energy_group = particle_energy(particle_info.particle_index); //Adjust Weights double new_weight = particle_info.weight*(1- (field_info.electron_density*sigma_ion)/ From 0fd0350dbee1f1c23848a08a24981f8502464de2 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 15:45:17 -0400 Subject: [PATCH 050/139] Change sigma to double from auto --- src/physics/degas2/DG2Physics.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 457ca84..61983c7 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -136,8 +136,8 @@ class DG2Physics { double energy = particle_energy(particle_info.particle_index); - auto sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); - auto sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); + double sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); + double sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); //Generate distance and move particle double l =-logf(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 @@ -162,8 +162,8 @@ class DG2Physics { double energy = particle_energy(particle_info.particle_index); - auto sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); - auto sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); + double sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); + double sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) From abd2fea4a1292892d86c47da669b24434eb1062b Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 15:46:17 -0400 Subject: [PATCH 051/139] Temporarily have weight output sigma_cx --- src/physics/degas2/DG2Physics.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 61983c7..18f6f1d 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -199,6 +199,7 @@ class DG2Physics { } } particle_info.weight = new_weight; + particle_info.weight = sigma_cx;//DELETE THIS } //To here random_pool_t random_pool; From 5757790dee71448b70b3311fc08885a3a01769a4 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 15:50:37 -0400 Subject: [PATCH 052/139] Temporarily have weight output sigma_cx inside the function that calculates it --- src/physics/degas2/DG2Physics.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 18f6f1d..54d8bca 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -122,6 +122,7 @@ class DG2Physics { } } double sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); + particle_info.weight = sigma_cx;//DELETE THIS return sigma_cx; } @@ -198,8 +199,8 @@ class DG2Physics { new_weight = ws; } } - particle_info.weight = new_weight; - particle_info.weight = sigma_cx;//DELETE THIS + //particle_info.weight = new_weight; + } //To here random_pool_t random_pool; From 8cba999a734f28cbde573353853ada1995459bf1 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 15:52:48 -0400 Subject: [PATCH 053/139] Weight Back to normal --- src/physics/degas2/DG2Physics.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 54d8bca..aee4ec1 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -122,7 +122,6 @@ class DG2Physics { } } double sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); - particle_info.weight = sigma_cx;//DELETE THIS return sigma_cx; } @@ -199,7 +198,7 @@ class DG2Physics { new_weight = ws; } } - //particle_info.weight = new_weight; + particle_info.weight = new_weight; } //To here From 6f20775eb1bc79a63ffcd85087db48c09006ec63 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 3 Jul 2025 15:55:18 -0400 Subject: [PATCH 054/139] Make the inputs be non-constant for the cross sections --- src/physics/degas2/DG2Physics.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index aee4ec1..a168264 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -52,7 +52,7 @@ class DG2Physics { } KOKKOS_FUNCTION - double ionization_cross_section(const double energy, const double e_temperature) const{ + double ionization_cross_section(double energy, double e_temperature) const{ double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 @@ -72,7 +72,7 @@ class DG2Physics { } KOKKOS_FUNCTION - double charge_exchange_cross_section(const double energy, const double ion_temperature) const{ + double charge_exchange_cross_section(double energy, double ion_temperature) const{ double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 From 7673d5b49116064aa2e3acec2261a63aaa894d60 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 08:56:29 -0400 Subject: [PATCH 055/139] Replace math functions with kokkos math functions --- src/physics/degas2/DG2Physics.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index a168264..27eb075 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -65,9 +65,9 @@ class DG2Physics { double lnrate_ion {0}; //cm^2 for (int i=0; i<9; i++) { - lnrate_ion += coef2[i]*std::pow(logf(e_temperature),i); + lnrate_ion += coef2[i]*Kokkos::pow(Kokkos::log(e_temperature),i); } - double sigma_ion = exp(lnrate_ion)/sqrt(particle_velocity_squared); + double sigma_ion = Kokkos::exp(lnrate_ion)/Kokkos::sqrt(particle_velocity_squared); return sigma_ion; } @@ -118,10 +118,10 @@ class DG2Physics { double lnrate_cx {0}; for (int i = 0; i < 9; i++) { for (int j = 0; j < 9; j++) { - lnrate_cx += coef[i][j]*std::pow(logf(energy),i)*std::pow(logf(ion_temperature),j); + lnrate_cx += coef[i][j]*Kokkos::pow(Kokkos::log(energy),i)*Kokkos::pow(Kokkos::log(ion_temperature),j); } } - double sigma_cx = std::exp(lnrate_cx)/sqrt(particle_velocity_squared); + double sigma_cx = Kokkos::exp(lnrate_cx)/Kokkos::sqrt(particle_velocity_squared); return sigma_cx; } @@ -140,7 +140,7 @@ class DG2Physics { double sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); //Generate distance and move particle - double l =-logf(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 + double l =-Kokkos::log(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 particle_info.position[0] += l*particle_info.direction[0]; particle_info.position[1] += l*particle_info.direction[1]; particle_info.position[2] += l*particle_info.direction[2]; @@ -169,18 +169,18 @@ class DG2Physics { //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - auto vx = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(x1))*cos(2*M_PI*x2); - auto vy = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(x1))*sin(2*M_PI*x2); - auto vz = std::sqrt(field_info.ion_temperature/mp)*std::sqrt(-2 * logf(y1))*sin(2*M_PI*y2); + auto vx = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::cos(2*M_PI*x2); + auto vy = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::sin(2*M_PI*x2); + auto vz = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(y1))*Kokkos::sin(2*M_PI*y2); - auto mag_v = std::sqrt(vx*vx + vy*vy + vz*vz); + auto mag_v = Kokkos::sqrt(vx*vx + vy*vy + vz*vz); particle_info.direction[0] = vx/mag_v; particle_info.direction[1] = vy/mag_v; particle_info.direction[2] = vz/mag_v; particle_energy(particle_info.particle_index) = 0.5*mp*mag_v*mag_v; - particle_info.energy_group = particle_energy(particle_info.particle_index); + particle_info.energy_group = particle_energy(particle_info.particle_index); //Temporary for debugging //Adjust Weights double new_weight = particle_info.weight*(1- (field_info.electron_density*sigma_ion)/ From 16c5bd74e32a0bf1a835b72e57d808c523136b33 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 09:02:25 -0400 Subject: [PATCH 056/139] temporarily have sigma_ion be just the energy and have that saved to weight to debug --- src/physics/degas2/DG2Physics.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 27eb075..e77b53a 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -68,7 +68,7 @@ class DG2Physics { lnrate_ion += coef2[i]*Kokkos::pow(Kokkos::log(e_temperature),i); } double sigma_ion = Kokkos::exp(lnrate_ion)/Kokkos::sqrt(particle_velocity_squared); - return sigma_ion; + return energy; } KOKKOS_FUNCTION @@ -198,7 +198,8 @@ class DG2Physics { new_weight = ws; } } - particle_info.weight = new_weight; + //particle_info.weight = new_weight; + particle_info.weight = sigma_ion; } //To here From 714e1861290db86db6c55f6e8ef1ca4714e4f2a7 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 09:10:48 -0400 Subject: [PATCH 057/139] pass the struct not the energy for sigma --- src/physics/degas2/DG2Physics.h | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index e77b53a..89f3f2e 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -52,7 +52,8 @@ class DG2Physics { } KOKKOS_FUNCTION - double ionization_cross_section(double energy, double e_temperature) const{ + double ionization_cross_section(const FieldInfo &field_info) const{ + double energy = particle_energy(particle_info.particle_index); double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 @@ -65,14 +66,15 @@ class DG2Physics { double lnrate_ion {0}; //cm^2 for (int i=0; i<9; i++) { - lnrate_ion += coef2[i]*Kokkos::pow(Kokkos::log(e_temperature),i); + lnrate_ion += coef2[i]*Kokkos::pow(Kokkos::log(field_info.electron_temperature),i); } double sigma_ion = Kokkos::exp(lnrate_ion)/Kokkos::sqrt(particle_velocity_squared); - return energy; + return sigma_ion; } KOKKOS_FUNCTION - double charge_exchange_cross_section(double energy, double ion_temperature) const{ + double charge_exchange_cross_section(const FieldInfo &field_info) const{ + double energy = particle_energy(particle_info.particle_index); double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 @@ -118,7 +120,7 @@ class DG2Physics { double lnrate_cx {0}; for (int i = 0; i < 9; i++) { for (int j = 0; j < 9; j++) { - lnrate_cx += coef[i][j]*Kokkos::pow(Kokkos::log(energy),i)*Kokkos::pow(Kokkos::log(ion_temperature),j); + lnrate_cx += coef[i][j]*Kokkos::pow(Kokkos::log(energy),i)*Kokkos::pow(Kokkos::log(field_info.ion_temperature),j); } } double sigma_cx = Kokkos::exp(lnrate_cx)/Kokkos::sqrt(particle_velocity_squared); @@ -134,10 +136,8 @@ class DG2Physics { double x = rand_gen.drand(0., 1.); random_pool.free_state(rand_gen); - double energy = particle_energy(particle_info.particle_index); - - double sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); - double sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); + double sigma_ion = ionization_cross_section(field_info); + double sigma_cx = charge_exchange_cross_section(field_info); //Generate distance and move particle double l =-Kokkos::log(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 @@ -160,10 +160,8 @@ class DG2Physics { double ww = rand_gen.drand(0., 1.); random_pool.free_state(rand_gen); - double energy = particle_energy(particle_info.particle_index); - - double sigma_ion = ionization_cross_section(energy, field_info.electron_temperature); - double sigma_cx = charge_exchange_cross_section(energy, field_info.ion_temperature); + double sigma_ion = ionization_cross_section(field_info); + double sigma_cx = charge_exchange_cross_section(field_info); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) From bccf642d5de6e92e1719f7bec6cc7cef1f42c851 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 09:15:02 -0400 Subject: [PATCH 058/139] pass the struct not the energy for sigma --- src/physics/degas2/DG2Physics.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 89f3f2e..e2aeaf1 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -52,7 +52,7 @@ class DG2Physics { } KOKKOS_FUNCTION - double ionization_cross_section(const FieldInfo &field_info) const{ + double ionization_cross_section(ParticleInfo &particle_info, const FieldInfo &field_info) const{ double energy = particle_energy(particle_info.particle_index); double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 @@ -73,7 +73,7 @@ class DG2Physics { } KOKKOS_FUNCTION - double charge_exchange_cross_section(const FieldInfo &field_info) const{ + double charge_exchange_cross_section(ParticleInfo &particle_info, const FieldInfo &field_info) const{ double energy = particle_energy(particle_info.particle_index); double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 @@ -136,8 +136,8 @@ class DG2Physics { double x = rand_gen.drand(0., 1.); random_pool.free_state(rand_gen); - double sigma_ion = ionization_cross_section(field_info); - double sigma_cx = charge_exchange_cross_section(field_info); + double sigma_ion = ionization_cross_section(particle_info, field_info); + double sigma_cx = charge_exchange_cross_section(particle_info, field_info); //Generate distance and move particle double l =-Kokkos::log(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 @@ -160,8 +160,8 @@ class DG2Physics { double ww = rand_gen.drand(0., 1.); random_pool.free_state(rand_gen); - double sigma_ion = ionization_cross_section(field_info); - double sigma_cx = charge_exchange_cross_section(field_info); + double sigma_ion = ionization_cross_section(particle_info, field_info); + double sigma_cx = charge_exchange_cross_section(particle_info, field_info); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) From b0e820ec02d759b18489940bd7c0565ffbba9842 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 09:18:05 -0400 Subject: [PATCH 059/139] replace sigma ion with energy again --- src/physics/degas2/DG2Physics.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index e2aeaf1..dede921 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -69,7 +69,7 @@ class DG2Physics { lnrate_ion += coef2[i]*Kokkos::pow(Kokkos::log(field_info.electron_temperature),i); } double sigma_ion = Kokkos::exp(lnrate_ion)/Kokkos::sqrt(particle_velocity_squared); - return sigma_ion; + return energy; } KOKKOS_FUNCTION From 945f542e2be9e6e64adb04df6ca8a1f9686eb3ab Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 09:25:52 -0400 Subject: [PATCH 060/139] Properly initialize particle energy --- src/physics/degas2/DG2Physics.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index dede921..26afa3e 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -38,6 +38,7 @@ class DG2Physics { // Initialize particle energy particle_energy = Kokkos::View("particle_energy", num_particles); + particle_energy(particle_info.particle_index) = particle_info.energy_group; //cross_sections = Kokkos::View("cross_sections", num_particles); } @@ -48,6 +49,7 @@ class DG2Physics { // Initialize particle energy particle_energy = Kokkos::View("particle_energy", num_particles); + particle_energy(particle_info.particle_index) = particle_info.energy_group; //cross_sections = Kokkos::View("cross_sections", num_particles); } @@ -69,7 +71,7 @@ class DG2Physics { lnrate_ion += coef2[i]*Kokkos::pow(Kokkos::log(field_info.electron_temperature),i); } double sigma_ion = Kokkos::exp(lnrate_ion)/Kokkos::sqrt(particle_velocity_squared); - return energy; + return sigma_ion; } KOKKOS_FUNCTION @@ -196,8 +198,7 @@ class DG2Physics { new_weight = ws; } } - //particle_info.weight = new_weight; - particle_info.weight = sigma_ion; + particle_info.weight = new_weight; } //To here From 13488dd9fef4b2c26371f221d38e6da4665af436 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 09:49:46 -0400 Subject: [PATCH 061/139] Initialize particle energy in a not very smart way. --- src/physics/degas2/DG2Physics.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 26afa3e..19899f7 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -38,7 +38,6 @@ class DG2Physics { // Initialize particle energy particle_energy = Kokkos::View("particle_energy", num_particles); - particle_energy(particle_info.particle_index) = particle_info.energy_group; //cross_sections = Kokkos::View("cross_sections", num_particles); } @@ -49,7 +48,6 @@ class DG2Physics { // Initialize particle energy particle_energy = Kokkos::View("particle_energy", num_particles); - particle_energy(particle_info.particle_index) = particle_info.energy_group; //cross_sections = Kokkos::View("cross_sections", num_particles); } @@ -67,6 +65,8 @@ class DG2Physics { coef2[6] = -2.631976175590e-3; coef2[7] = 1.119543953861e-4; coef2[8] = -2.039149852002e-6; double lnrate_ion {0}; //cm^2 + + //Should this be a parallel_for? for (int i=0; i<9; i++) { lnrate_ion += coef2[i]*Kokkos::pow(Kokkos::log(field_info.electron_temperature),i); } @@ -138,6 +138,14 @@ class DG2Physics { double x = rand_gen.drand(0., 1.); random_pool.free_state(rand_gen); + //This is really not a good way to do this. Initializes the particle energy view if it is ~0 from the energy gorups + //where I currently store energy. In principle this should work to initialize from any source but you would + //need to pass it. I suspect the solution is either add a seperate energy initialization function or add an input + //into the constructor and have it initialize there. You'd need the index and the eenrgy to initialize. + if (particle_energy(particle_info.particle_index) < 1e-6){ + particle_energy(particle_info.particle_index) = particle_info.energy_group; + } + double sigma_ion = ionization_cross_section(particle_info, field_info); double sigma_cx = charge_exchange_cross_section(particle_info, field_info); From c35e40932b346b41f1c1bba7a7b45e9224feb040 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 10:31:57 -0400 Subject: [PATCH 062/139] Print average of distances --- test/test_degas2_physics.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 1e06b39..b363a76 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -54,6 +54,15 @@ int main(int argc, char *argv[]) { auto output = create_mirror_view(particles); Kokkos::deep_copy(output, particles); std::ofstream outfile("Log.txt"); + + double l {0}; + for (int i; i < numParticles; ++i) { + l += output(i).position[0]; + } + l /= numParticles; + outfile << "Mean free path (cm): " << l << std::endl; + + outfile << "Particle #,Energy(eV),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; for (int i = 0; i < numParticles; ++i) { From 8e23013763d169cb6bda72bfd5697dd40a94fa1d Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 13:30:57 -0400 Subject: [PATCH 063/139] Print some statistics on x direction after collision and mean free path. --- test/test_degas2_physics.cpp | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index b363a76..ec7256e 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -60,7 +60,30 @@ int main(int argc, char *argv[]) { l += output(i).position[0]; } l /= numParticles; - outfile << "Mean free path (cm): " << l << std::endl; + outfile << "Average Distance (cm): " << l << std::endl; + + double varl {0}; + for (int i; i < numParticles; ++i) { + varl += (output(i).position[0] - l)*(output(i).position[0] - l); + } + varl /= (numParticles - 1); + double sdl {sqrt(varl)}; + outfile << "Standard Deviation of Distance (cm): " << sdl << std::endl; + + double ux {0}; + for (int i; i < numParticles; ++i) { + ux += output(i).direction[0]; + } + ux /= numParticles; + outfile << "Mean x Direction: " << ux << std::endl; + + double varux + for (int i; i < numParticles; ++i) { + varux += (output(i).direction[0] - ux)*(output(i).direction[0] - ux); + } + varux /= (numParticles - 1); + double sdux {sqrt(varux)}; + outfile << "Standard Deviation of Mean x Direction: " << sdux << std::endl; outfile << "Particle #,Energy(eV),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; From 1aa82417c50418b599b81c6ac9c47dcdda247ccf Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 13:33:21 -0400 Subject: [PATCH 064/139] Set loop variables initial value --- test/test_degas2_physics.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index ec7256e..737b876 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -56,14 +56,14 @@ int main(int argc, char *argv[]) { std::ofstream outfile("Log.txt"); double l {0}; - for (int i; i < numParticles; ++i) { + for (int i=0; i < numParticles; ++i) { l += output(i).position[0]; } l /= numParticles; outfile << "Average Distance (cm): " << l << std::endl; double varl {0}; - for (int i; i < numParticles; ++i) { + for (int i=0; i < numParticles; ++i) { varl += (output(i).position[0] - l)*(output(i).position[0] - l); } varl /= (numParticles - 1); @@ -71,14 +71,14 @@ int main(int argc, char *argv[]) { outfile << "Standard Deviation of Distance (cm): " << sdl << std::endl; double ux {0}; - for (int i; i < numParticles; ++i) { + for (int i=0; i < numParticles; ++i) { ux += output(i).direction[0]; } ux /= numParticles; outfile << "Mean x Direction: " << ux << std::endl; double varux - for (int i; i < numParticles; ++i) { + for (int i=0; i < numParticles; ++i) { varux += (output(i).direction[0] - ux)*(output(i).direction[0] - ux); } varux /= (numParticles - 1); From 7b520ab0e91fc33b5152e415a2c68a6c3b760d00 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 13:33:21 -0400 Subject: [PATCH 065/139] Set loop variables initial value --- test/test_degas2_physics.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index ec7256e..c3d8fb0 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -56,14 +56,14 @@ int main(int argc, char *argv[]) { std::ofstream outfile("Log.txt"); double l {0}; - for (int i; i < numParticles; ++i) { + for (int i=0; i < numParticles; ++i) { l += output(i).position[0]; } l /= numParticles; outfile << "Average Distance (cm): " << l << std::endl; double varl {0}; - for (int i; i < numParticles; ++i) { + for (int i=0; i < numParticles; ++i) { varl += (output(i).position[0] - l)*(output(i).position[0] - l); } varl /= (numParticles - 1); @@ -71,14 +71,14 @@ int main(int argc, char *argv[]) { outfile << "Standard Deviation of Distance (cm): " << sdl << std::endl; double ux {0}; - for (int i; i < numParticles; ++i) { + for (int i=0; i < numParticles; ++i) { ux += output(i).direction[0]; } ux /= numParticles; outfile << "Mean x Direction: " << ux << std::endl; - double varux - for (int i; i < numParticles; ++i) { + double varux; + for (int i=0; i < numParticles; ++i) { varux += (output(i).direction[0] - ux)*(output(i).direction[0] - ux); } varux /= (numParticles - 1); From 8aa6171b9f9e4eb8ec65a30a0a657ffde4b71edb Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 14:06:22 -0400 Subject: [PATCH 066/139] Define set particle energies function --- src/physics/degas2/DG2Physics.h | 9 +++++++-- test/test_degas2_physics.cpp | 19 ++++++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 19899f7..f2c8dae 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -51,6 +51,11 @@ class DG2Physics { //cross_sections = Kokkos::View("cross_sections", num_particles); } + KOKKOS_FUNCTION + void set_energy(ParticleInfo &particle_info, const double energy) const{ + particle_energy(particle_info.particle_index) = energy; + } + KOKKOS_FUNCTION double ionization_cross_section(ParticleInfo &particle_info, const FieldInfo &field_info) const{ double energy = particle_energy(particle_info.particle_index); @@ -142,9 +147,9 @@ class DG2Physics { //where I currently store energy. In principle this should work to initialize from any source but you would //need to pass it. I suspect the solution is either add a seperate energy initialization function or add an input //into the constructor and have it initialize there. You'd need the index and the eenrgy to initialize. - if (particle_energy(particle_info.particle_index) < 1e-6){ + /*if (particle_energy(particle_info.particle_index) < 1e-6){ particle_energy(particle_info.particle_index) = particle_info.energy_group; - } + }*/ double sigma_ion = ionization_cross_section(particle_info, field_info); double sigma_cx = charge_exchange_cross_section(particle_info, field_info); diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index c3d8fb0..e51d532 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -15,6 +15,7 @@ int main(int argc, char *argv[]) { { int numParticles = 1000; + double energy = 3.0; Kokkos::View sigma_t_; // mat, T, g Kokkos::View sigma_a_; // mat, T, g @@ -34,7 +35,7 @@ int main(int argc, char *argv[]) { particles(i).direction[0] = 1.0; particles(i).direction[1] = 0; particles(i).direction[2] = 0; - particles(i).energy_group = 3.0; + particles(i).energy_group = energy; particles(i).weight = 1.0; particles(i).particle_index = i; @@ -45,6 +46,12 @@ int main(int argc, char *argv[]) { // and others like this }); + // Set the particle energy view + Kokkos::parallel_for( + "Set energies", numParticles, KOKKOS_LAMBDA(int i) { + physics.set_energy(particles(i), energy); + }); + // Call the function to be tested Kokkos::parallel_for( "run physics", numParticles, KOKKOS_LAMBDA(int i) { @@ -55,12 +62,14 @@ int main(int argc, char *argv[]) { Kokkos::deep_copy(output, particles); std::ofstream outfile("Log.txt"); + //These 4 functions should return the commented value for SEED=12345 + double l {0}; for (int i=0; i < numParticles; ++i) { l += output(i).position[0]; } l /= numParticles; - outfile << "Average Distance (cm): " << l << std::endl; + outfile << "Average Distance (cm): " << l << std::endl; //2.41 double varl {0}; for (int i=0; i < numParticles; ++i) { @@ -68,14 +77,14 @@ int main(int argc, char *argv[]) { } varl /= (numParticles - 1); double sdl {sqrt(varl)}; - outfile << "Standard Deviation of Distance (cm): " << sdl << std::endl; + outfile << "Standard Deviation of Distance (cm): " << sdl << std::endl; //2.38 double ux {0}; for (int i=0; i < numParticles; ++i) { ux += output(i).direction[0]; } ux /= numParticles; - outfile << "Mean x Direction: " << ux << std::endl; + outfile << "Mean x Direction: " << ux << std::endl; //0.0159 double varux; for (int i=0; i < numParticles; ++i) { @@ -83,7 +92,7 @@ int main(int argc, char *argv[]) { } varux /= (numParticles - 1); double sdux {sqrt(varux)}; - outfile << "Standard Deviation of Mean x Direction: " << sdux << std::endl; + outfile << "Standard Deviation of Mean x Direction: " << sdux << std::endl; //0.579 outfile << "Particle #,Energy(eV),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; From 6c4bd93506646f0bd6260abb63da1c6bd548eead Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 14:24:03 -0400 Subject: [PATCH 067/139] Add catch2 checks --- test/test_degas2_physics.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index e51d532..506958e 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -109,7 +109,10 @@ int main(int argc, char *argv[]) { outfile << output(i).direction[2] << std::endl; } - + REQUIRE_THAT(l, WithinAbs(2.40,.03)) + REQUIRE_THAT(sdl, WithinAbs(2.40,.03)) + REQUIRE_THAT(ux, WithinAbs(0.0,.03)) + REQUIRE_THAT(sdx, WithinAbs(0.577,.03)) } Kokkos::finalize(); return 0; From 5dc49d1933339ec5ffdea776ceea1919b04b994f Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 14:25:29 -0400 Subject: [PATCH 068/139] Add catch2 checks --- test/test_degas2_physics.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 506958e..10e48cb 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -8,6 +8,7 @@ #include "DG2Physics.h" #include #include +#include int main(int argc, char *argv[]) { @@ -109,10 +110,10 @@ int main(int argc, char *argv[]) { outfile << output(i).direction[2] << std::endl; } - REQUIRE_THAT(l, WithinAbs(2.40,.03)) - REQUIRE_THAT(sdl, WithinAbs(2.40,.03)) - REQUIRE_THAT(ux, WithinAbs(0.0,.03)) - REQUIRE_THAT(sdx, WithinAbs(0.577,.03)) + REQUIRE_THAT(l, WithinAbs(2.40,.03)); + REQUIRE_THAT(sdl, WithinAbs(2.40,.03)); + REQUIRE_THAT(ux, WithinAbs(0.0,.03)); + REQUIRE_THAT(sdx, WithinAbs(0.577,.03)); } Kokkos::finalize(); return 0; From a097511067459f157deaed583d08a2839d00b1c3 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 14:29:59 -0400 Subject: [PATCH 069/139] Add Catch:: namespace --- test/test_degas2_physics.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 10e48cb..8fad84f 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -110,10 +110,10 @@ int main(int argc, char *argv[]) { outfile << output(i).direction[2] << std::endl; } - REQUIRE_THAT(l, WithinAbs(2.40,.03)); - REQUIRE_THAT(sdl, WithinAbs(2.40,.03)); - REQUIRE_THAT(ux, WithinAbs(0.0,.03)); - REQUIRE_THAT(sdx, WithinAbs(0.577,.03)); + REQUIRE_THAT(l, Catch::WithinAbs(2.40,.03)); + REQUIRE_THAT(sdl, Catch::WithinAbs(2.40,.03)); + REQUIRE_THAT(ux, Catch::WithinAbs(0.0,.03)); + REQUIRE_THAT(sdux, Catch::WithinAbs(0.577,.03)); } Kokkos::finalize(); return 0; From 739453a1cd0c6d0f1ab8a234aeb305ea1eead6ed Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 14:31:25 -0400 Subject: [PATCH 070/139] Add Catch::Matchers:: namespace --- test/test_degas2_physics.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 8fad84f..ba96e09 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -110,10 +110,10 @@ int main(int argc, char *argv[]) { outfile << output(i).direction[2] << std::endl; } - REQUIRE_THAT(l, Catch::WithinAbs(2.40,.03)); - REQUIRE_THAT(sdl, Catch::WithinAbs(2.40,.03)); - REQUIRE_THAT(ux, Catch::WithinAbs(0.0,.03)); - REQUIRE_THAT(sdux, Catch::WithinAbs(0.577,.03)); + REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.03)); + REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); + REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); + REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); } Kokkos::finalize(); return 0; From d3c7551b6d938fe760cf930213d27999a6e2dab4 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 15:04:50 -0400 Subject: [PATCH 071/139] try moving tests outside of Kokkos::finalize --- test/test_degas2_physics.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index ba96e09..ab4243f 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -110,11 +110,12 @@ int main(int argc, char *argv[]) { outfile << output(i).direction[2] << std::endl; } - REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.03)); - REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); - REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); - REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); + } Kokkos::finalize(); + REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.03)); + REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); + REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); + REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); return 0; } From 090ded53bf38dc3be7f03fc2d48c65a64d62994a Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 15:09:13 -0400 Subject: [PATCH 072/139] return require to befre kokkos finalize --- test/test_degas2_physics.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index ab4243f..6db8bb5 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -109,13 +109,13 @@ int main(int argc, char *argv[]) { outfile << output(i).direction[1] << ","; outfile << output(i).direction[2] << std::endl; } - + REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.03)); + REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); + REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); + REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); } Kokkos::finalize(); - REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.03)); - REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); - REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); - REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); + return 0; } From 13c4e240ea4bc283d68a68813568128ab9bf3d68 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 15:55:37 -0400 Subject: [PATCH 073/139] Convert main() to TEST_CASE --- test/CMakeLists.txt | 2 +- test/test_degas2_physics.cpp | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f5c1995..3d3ef1b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -32,7 +32,7 @@ catch_discover_tests(test_pumi_tally_impl_methods) if(PUMI_ENABLE_DEGAS2) add_executable(test_degas2_physics test_degas2_physics.cpp) target_include_directories(test_degas2_physics PRIVATE ${PROJECT_SOURCE_DIR}/src/physics/degas2) - target_link_libraries(test_degas2_physics PUBLIC Catch2::Catch2 pumitally_degas2) + target_link_libraries(test_degas2_physics PUBLIC Catch2::Catch2WithMain pumitally_degas2) catch_discover_tests(test_degas2_physics DISCOVERY_MODE PRE_TEST) endif() #################### Test HostWrite #################### \ No newline at end of file diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 6db8bb5..97bd0d8 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -10,9 +10,8 @@ #include #include - -int main(int argc, char *argv[]) { - Kokkos::initialize(argc, argv); +TEST_CASE("Test Degas2 Physics Functions"){ + Kokkos::initialize(); { int numParticles = 1000; @@ -117,5 +116,4 @@ int main(int argc, char *argv[]) { } Kokkos::finalize(); - return 0; } From 7ab2a5f31922da00b565a0cb291cadf7aa36e1d8 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 15:59:58 -0400 Subject: [PATCH 074/139] Make the test fail to confirm it works --- test/test_degas2_physics.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 97bd0d8..431c671 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -112,6 +112,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); + REQUIRE_THAT(5.0, Catch::Matchers::WithinAbs(1.0,.03)); } Kokkos::finalize(); From 6f81472e195f3f36428197fd941fc0b9d46db22e Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 7 Jul 2025 16:03:18 -0400 Subject: [PATCH 075/139] Return to functional tests --- test/test_degas2_physics.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 431c671..c0406b1 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -112,8 +112,6 @@ TEST_CASE("Test Degas2 Physics Functions"){ REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); - REQUIRE_THAT(5.0, Catch::Matchers::WithinAbs(1.0,.03)); - } Kokkos::finalize(); From 818b88de57dfad0d4e911bf79708d3317ff69d49 Mon Sep 17 00:00:00 2001 From: Gabe Date: Tue, 8 Jul 2025 14:24:49 -0400 Subject: [PATCH 076/139] Comment out logging --- test/test_degas2_physics.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index c0406b1..9279c88 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -60,7 +60,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ }); auto output = create_mirror_view(particles); Kokkos::deep_copy(output, particles); - std::ofstream outfile("Log.txt"); + //std::ofstream outfile("Log.txt"); //These 4 functions should return the commented value for SEED=12345 @@ -69,7 +69,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ l += output(i).position[0]; } l /= numParticles; - outfile << "Average Distance (cm): " << l << std::endl; //2.41 + //outfile << "Average Distance (cm): " << l << std::endl; //2.41 double varl {0}; for (int i=0; i < numParticles; ++i) { @@ -77,14 +77,14 @@ TEST_CASE("Test Degas2 Physics Functions"){ } varl /= (numParticles - 1); double sdl {sqrt(varl)}; - outfile << "Standard Deviation of Distance (cm): " << sdl << std::endl; //2.38 + //outfile << "Standard Deviation of Distance (cm): " << sdl << std::endl; //2.38 double ux {0}; for (int i=0; i < numParticles; ++i) { ux += output(i).direction[0]; } ux /= numParticles; - outfile << "Mean x Direction: " << ux << std::endl; //0.0159 + //outfile << "Mean x Direction: " << ux << std::endl; //0.0159 double varux; for (int i=0; i < numParticles; ++i) { @@ -92,6 +92,8 @@ TEST_CASE("Test Degas2 Physics Functions"){ } varux /= (numParticles - 1); double sdux {sqrt(varux)}; + + /* outfile << "Standard Deviation of Mean x Direction: " << sdux << std::endl; //0.579 @@ -108,6 +110,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ outfile << output(i).direction[1] << ","; outfile << output(i).direction[2] << std::endl; } + */ REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.03)); REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); From 05fd1b2fe0c391b560729748c39b5ab398fd73eb Mon Sep 17 00:00:00 2001 From: Gabe Date: Tue, 8 Jul 2025 14:30:16 -0400 Subject: [PATCH 077/139] delete unneeded comment --- src/physics/degas2/DG2Physics.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index f2c8dae..e73717c 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -71,7 +71,6 @@ class DG2Physics { double lnrate_ion {0}; //cm^2 - //Should this be a parallel_for? for (int i=0; i<9; i++) { lnrate_ion += coef2[i]*Kokkos::pow(Kokkos::log(field_info.electron_temperature),i); } @@ -143,14 +142,6 @@ class DG2Physics { double x = rand_gen.drand(0., 1.); random_pool.free_state(rand_gen); - //This is really not a good way to do this. Initializes the particle energy view if it is ~0 from the energy gorups - //where I currently store energy. In principle this should work to initialize from any source but you would - //need to pass it. I suspect the solution is either add a seperate energy initialization function or add an input - //into the constructor and have it initialize there. You'd need the index and the eenrgy to initialize. - /*if (particle_energy(particle_info.particle_index) < 1e-6){ - particle_energy(particle_info.particle_index) = particle_info.energy_group; - }*/ - double sigma_ion = ionization_cross_section(particle_info, field_info); double sigma_cx = charge_exchange_cross_section(particle_info, field_info); From 7141a5c874b8a5f9178e3f788d6d778bbb5cde6b Mon Sep 17 00:00:00 2001 From: Gabe Date: Tue, 8 Jul 2025 14:37:57 -0400 Subject: [PATCH 078/139] Change energy group back to int and stop assigning it to the energy --- src/physics/degas2/DG2Physics.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index e73717c..fb47cd8 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -18,7 +18,7 @@ struct ParticleInfo { double position[3]; // Position in space (x, y, z) double direction[3]; // Direction vector (unit vector) double weight; - double energy_group; // Energy group *index* + int energy_group; // Energy group *index* int particle_index; }; @@ -184,7 +184,7 @@ class DG2Physics { particle_info.direction[2] = vz/mag_v; particle_energy(particle_info.particle_index) = 0.5*mp*mag_v*mag_v; - particle_info.energy_group = particle_energy(particle_info.particle_index); //Temporary for debugging + //particle_info.energy_group = particle_energy(particle_info.particle_index); //Temporary for debugging //Adjust Weights double new_weight = particle_info.weight*(1- (field_info.electron_density*sigma_ion)/ From e4e70a2f56e0f11280088cf9d921218fd6dc8bfe Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 09:57:54 -0400 Subject: [PATCH 079/139] Make a test that tracks one particle until death --- test/test_degas2_physics.cpp | 91 ++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 9279c88..690e081 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -119,3 +119,94 @@ TEST_CASE("Test Degas2 Physics Functions"){ Kokkos::finalize(); } + +TEST_CASE("Test Degas2 Physics Functions"){ + Kokkos::initialize(); + + { + int numParticles = 1; + double energy = 3.0; + + Kokkos::View sigma_t_; // mat, T, g + Kokkos::View sigma_a_; // mat, T, g + Kokkos::View scattering_matrix_; // mat, T, g, g + Kokkos::View sigma_s_; // mat, T, g + DG2CrossSection crossSection(sigma_t_, sigma_a_, scattering_matrix_, + sigma_s_); + DG2Physics physics(crossSection, numParticles); + + Kokkos::View particles("particles", numParticles); + Kokkos::View fields("fields", numParticles); + Kokkos::View track("track", 1); + + Kokkos::parallel_for( + "initialize_particles", numParticles, KOKKOS_LAMBDA(int i) { + particles(i).position[0] = 0; + particles(i).position[1] = 0; + particles(i).position[2] = 0; + particles(i).direction[0] = 1.0; + particles(i).direction[1] = 0; + particles(i).direction[2] = 0; + particles(i).energy_group = energy; + particles(i).weight = 1.0; + particles(i).particle_index = i; + + fields(i).electron_density = 1.0e13; + fields(i).ion_density = 1.0e13; + fields(i).electron_temperature = 1000.0; + fields(i).ion_temperature = 1000.0; + // and others like this + }); + + // Set the particle energy view and randomize direction and energy under the Maxwelllian + Kokkos::parallel_for( + "Set energies", numParticles, KOKKOS_LAMBDA(int i) { + physics.set_energy(particles(i), energy); + physics.collide_particles(particles(i), fields(i)); + }); + + //Reset weight + Kokkos::parallel_for( + "initialize_particles", numParticles, KOKKOS_LAMBDA(int i) { + particles(i).weight = 1.0; + }); + + // Call the function to be tested + Kokkos::parallel_for( + "run physics", numParticles, KOKKOS_LAMBDA(int i) { + int j = 0 + while(particle(i).particle_weight > 0.0001){ + if j != 0{ + Kokkos::resize(track, j+1); + } + physics.sample_collision_distance(particles(i), fields(i)); + physics.collide_particle(particles(i), fields(i)); + track(j) = particles(i); + j++; + } + }); + auto output = create_mirror_view(track); + Kokkos::deep_copy(output, track); + + int length = output.extent(0); + + std::ofstream outfile("Log.txt"); + outfile << "Particle #,Energy(eV),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; + + for (int i = 0; i < length; ++i) { + outfile << i << ","; + outfile << output(i).energy_group << ","; + outfile << output(i).weight << ","; + outfile << output(i).position[0] << ","; + outfile << output(i).position[1] << ","; + outfile << output(i).position[2] << ","; + outfile << output(i).direction[0] << ","; + outfile << output(i).direction[1] << ","; + outfile << output(i).direction[2] << std::endl; + } + REQUIRE_THAT(output(length-1), Catch::Matchers::WithinAbs(0,.03)); + + } + Kokkos::finalize(); + +} \ No newline at end of file From ec902e41bf9f7e0eeb1b2609be4b21586d4a59aa Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 10:20:20 -0400 Subject: [PATCH 080/139] Make a test that tracks one particle until death --- test/test_degas2_physics.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 690e081..d3d68fe 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -162,7 +162,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ Kokkos::parallel_for( "Set energies", numParticles, KOKKOS_LAMBDA(int i) { physics.set_energy(particles(i), energy); - physics.collide_particles(particles(i), fields(i)); + physics.collide_particle(particles(i), fields(i)); }); //Reset weight @@ -174,7 +174,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ // Call the function to be tested Kokkos::parallel_for( "run physics", numParticles, KOKKOS_LAMBDA(int i) { - int j = 0 + int j = 0; while(particle(i).particle_weight > 0.0001){ if j != 0{ Kokkos::resize(track, j+1); @@ -204,7 +204,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ outfile << output(i).direction[1] << ","; outfile << output(i).direction[2] << std::endl; } - REQUIRE_THAT(output(length-1), Catch::Matchers::WithinAbs(0,.03)); + REQUIRE_THAT(output(length-1).weight, Catch::Matchers::WithinAbs(0,.03)); } Kokkos::finalize(); From 5c612563f4ee63092dbe1cd6a61576678a052c35 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 10:23:43 -0400 Subject: [PATCH 081/139] Make a test that tracks one particle until death --- test/test_degas2_physics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index d3d68fe..987cafa 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -175,8 +175,8 @@ TEST_CASE("Test Degas2 Physics Functions"){ Kokkos::parallel_for( "run physics", numParticles, KOKKOS_LAMBDA(int i) { int j = 0; - while(particle(i).particle_weight > 0.0001){ - if j != 0{ + while(particles(i).particle_weight > 0.0001){ + if (j != 0){ Kokkos::resize(track, j+1); } physics.sample_collision_distance(particles(i), fields(i)); From f20065ba72346b5efa6f934694f7c50dc5810cbb Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 10:28:53 -0400 Subject: [PATCH 082/139] Make a test that tracks one particle until death --- test/test_degas2_physics.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 987cafa..2cc59fc 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -120,7 +120,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ } -TEST_CASE("Test Degas2 Physics Functions"){ +TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ Kokkos::initialize(); { @@ -175,9 +175,9 @@ TEST_CASE("Test Degas2 Physics Functions"){ Kokkos::parallel_for( "run physics", numParticles, KOKKOS_LAMBDA(int i) { int j = 0; - while(particles(i).particle_weight > 0.0001){ + while(particles(i).weight > 0.0001){ if (j != 0){ - Kokkos::resize(track, j+1); + Kokkos::resize(track, (j+1)); } physics.sample_collision_distance(particles(i), fields(i)); physics.collide_particle(particles(i), fields(i)); From 152951abfc6cf94aa44026a3e5bb1aba4e1957b3 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 10:30:47 -0400 Subject: [PATCH 083/139] Make a test that tracks one particle until death --- test/test_degas2_physics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 2cc59fc..4aee5b2 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -177,7 +177,7 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ int j = 0; while(particles(i).weight > 0.0001){ if (j != 0){ - Kokkos::resize(track, (j+1)); + Kokkos::resize(track, const (j+1)); } physics.sample_collision_distance(particles(i), fields(i)); physics.collide_particle(particles(i), fields(i)); From ccf0d7f67417225397fad6452764335d0eae8a37 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 10:33:32 -0400 Subject: [PATCH 084/139] Make a test that tracks one particle until death --- test/test_degas2_physics.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 4aee5b2..0527be5 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -174,10 +174,10 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ // Call the function to be tested Kokkos::parallel_for( "run physics", numParticles, KOKKOS_LAMBDA(int i) { - int j = 0; + int j = 1; while(particles(i).weight > 0.0001){ - if (j != 0){ - Kokkos::resize(track, const (j+1)); + if (j != 1){ + Kokkos::resize(track, j); } physics.sample_collision_distance(particles(i), fields(i)); physics.collide_particle(particles(i), fields(i)); From f831c015b33641b9f69a40407a0febd04c0aba83 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 10:44:21 -0400 Subject: [PATCH 085/139] Make a test that tracks one particle until death --- test/test_degas2_physics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 0527be5..ff8cfe1 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -177,7 +177,7 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ int j = 1; while(particles(i).weight > 0.0001){ if (j != 1){ - Kokkos::resize(track, j); + Kokkos::resize(track, static_cast(j)); } physics.sample_collision_distance(particles(i), fields(i)); physics.collide_particle(particles(i), fields(i)); @@ -188,7 +188,7 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ auto output = create_mirror_view(track); Kokkos::deep_copy(output, track); - int length = output.extent(0); + const int length = output.extent(0); std::ofstream outfile("Log.txt"); outfile << "Particle #,Energy(eV),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; From 4161d6a747e57a1b22351f328eb41579872bef5f Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 10:44:21 -0400 Subject: [PATCH 086/139] Make a test that tracks one particle until death --- test/test_degas2_physics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 0527be5..ff8cfe1 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -177,7 +177,7 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ int j = 1; while(particles(i).weight > 0.0001){ if (j != 1){ - Kokkos::resize(track, j); + Kokkos::resize(track, static_cast(j)); } physics.sample_collision_distance(particles(i), fields(i)); physics.collide_particle(particles(i), fields(i)); @@ -188,7 +188,7 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ auto output = create_mirror_view(track); Kokkos::deep_copy(output, track); - int length = output.extent(0); + const int length = output.extent(0); std::ofstream outfile("Log.txt"); outfile << "Particle #,Energy(eV),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; From d7180c908ff85640bc3ffb9e53976125ee6cf7ff Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 10:45:39 -0400 Subject: [PATCH 087/139] Make a test that tracks one particle until death --- test/test_degas2_physics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index ff8cfe1..334de97 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -177,7 +177,7 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ int j = 1; while(particles(i).weight > 0.0001){ if (j != 1){ - Kokkos::resize(track, static_cast(j)); + Kokkos::resize(track, static_cast(j)); } physics.sample_collision_distance(particles(i), fields(i)); physics.collide_particle(particles(i), fields(i)); From 3dae7125692c2fa2c6ee464d1799bf22c9196796 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 10:53:50 -0400 Subject: [PATCH 088/139] Make a test that tracks one particle until death --- test/test_degas2_physics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 334de97..a6884ec 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -177,7 +177,7 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ int j = 1; while(particles(i).weight > 0.0001){ if (j != 1){ - Kokkos::resize(track, static_cast(j)); + Kokkos::resize(track, (track.extent(0)+1)); } physics.sample_collision_distance(particles(i), fields(i)); physics.collide_particle(particles(i), fields(i)); From 742bf0f24e189eb74e89063392abd8cb5d9a7383 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 11:00:58 -0400 Subject: [PATCH 089/139] Test avoiding the custom type --- test/test_degas2_physics.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index a6884ec..858e734 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -137,7 +137,7 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ Kokkos::View particles("particles", numParticles); Kokkos::View fields("fields", numParticles); - Kokkos::View track("track", 1); + Kokkos::View pos("pos", 1); Kokkos::parallel_for( "initialize_particles", numParticles, KOKKOS_LAMBDA(int i) { @@ -177,11 +177,13 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ int j = 1; while(particles(i).weight > 0.0001){ if (j != 1){ - Kokkos::resize(track, (track.extent(0)+1)); + Kokkos::resize(pos, (pos.extent(0)+1)); } physics.sample_collision_distance(particles(i), fields(i)); physics.collide_particle(particles(i), fields(i)); - track(j) = particles(i); + pos(j,0) = particles(i).position[0]; + pos(j,1) = particles(i).position[1]; + pos(j,2) = particles(i).position[2]; j++; } }); From d7a5d6f796ccfad2476ab3560279b09ec675bc74 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 11:05:50 -0400 Subject: [PATCH 090/139] Return to particle_info type. Just make the array way to large and leave the extra blank? --- test/test_degas2_physics.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 858e734..f9f0167 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -137,7 +137,7 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ Kokkos::View particles("particles", numParticles); Kokkos::View fields("fields", numParticles); - Kokkos::View pos("pos", 1); + Kokkos::View track("track", 1000); Kokkos::parallel_for( "initialize_particles", numParticles, KOKKOS_LAMBDA(int i) { @@ -174,16 +174,14 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ // Call the function to be tested Kokkos::parallel_for( "run physics", numParticles, KOKKOS_LAMBDA(int i) { - int j = 1; + int j = 0; while(particles(i).weight > 0.0001){ - if (j != 1){ - Kokkos::resize(pos, (pos.extent(0)+1)); - } + //if (j != 0){ + // Kokkos::resize(track, (track.extent(0)+1)); + //} physics.sample_collision_distance(particles(i), fields(i)); physics.collide_particle(particles(i), fields(i)); - pos(j,0) = particles(i).position[0]; - pos(j,1) = particles(i).position[1]; - pos(j,2) = particles(i).position[2]; + track(j) = particles(i); j++; } }); From 59dbdabbe8c5e6a52e8539f0f20af702fa37cfe2 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 11:10:51 -0400 Subject: [PATCH 091/139] Return to particle_info type. Just make the array way to large and leave the extra blank? --- test/test_degas2_physics.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index f9f0167..0d37d95 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -9,7 +9,7 @@ #include #include #include - +/* TEST_CASE("Test Degas2 Physics Functions"){ Kokkos::initialize(); @@ -110,7 +110,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ outfile << output(i).direction[1] << ","; outfile << output(i).direction[2] << std::endl; } - */ + REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.03)); REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); @@ -119,7 +119,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ Kokkos::finalize(); } - +*/ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ Kokkos::initialize(); From df805b64a7cfcf37c89589573f1d4c992ed27251 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 11:16:39 -0400 Subject: [PATCH 092/139] Change seed --- test/test_degas2_physics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 0d37d95..dea0c64 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -133,7 +133,7 @@ TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ Kokkos::View sigma_s_; // mat, T, g DG2CrossSection crossSection(sigma_t_, sigma_a_, scattering_matrix_, sigma_s_); - DG2Physics physics(crossSection, numParticles); + DG2Physics physics(crossSection, numParticles, 12346); Kokkos::View particles("particles", numParticles); Kokkos::View fields("fields", numParticles); From 60276e070e31641e1fa92f9f98aca6ae97452ffb Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 9 Jul 2025 11:28:20 -0400 Subject: [PATCH 093/139] Revert to last finished form --- test/test_degas2_physics.cpp | 95 +----------------------------------- 1 file changed, 2 insertions(+), 93 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index dea0c64..9279c88 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -9,7 +9,7 @@ #include #include #include -/* + TEST_CASE("Test Degas2 Physics Functions"){ Kokkos::initialize(); @@ -110,7 +110,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ outfile << output(i).direction[1] << ","; outfile << output(i).direction[2] << std::endl; } - + */ REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.03)); REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); @@ -119,94 +119,3 @@ TEST_CASE("Test Degas2 Physics Functions"){ Kokkos::finalize(); } -*/ -TEST_CASE("Test Degas2 Physics Particle Track Until Destroyed"){ - Kokkos::initialize(); - - { - int numParticles = 1; - double energy = 3.0; - - Kokkos::View sigma_t_; // mat, T, g - Kokkos::View sigma_a_; // mat, T, g - Kokkos::View scattering_matrix_; // mat, T, g, g - Kokkos::View sigma_s_; // mat, T, g - DG2CrossSection crossSection(sigma_t_, sigma_a_, scattering_matrix_, - sigma_s_); - DG2Physics physics(crossSection, numParticles, 12346); - - Kokkos::View particles("particles", numParticles); - Kokkos::View fields("fields", numParticles); - Kokkos::View track("track", 1000); - - Kokkos::parallel_for( - "initialize_particles", numParticles, KOKKOS_LAMBDA(int i) { - particles(i).position[0] = 0; - particles(i).position[1] = 0; - particles(i).position[2] = 0; - particles(i).direction[0] = 1.0; - particles(i).direction[1] = 0; - particles(i).direction[2] = 0; - particles(i).energy_group = energy; - particles(i).weight = 1.0; - particles(i).particle_index = i; - - fields(i).electron_density = 1.0e13; - fields(i).ion_density = 1.0e13; - fields(i).electron_temperature = 1000.0; - fields(i).ion_temperature = 1000.0; - // and others like this - }); - - // Set the particle energy view and randomize direction and energy under the Maxwelllian - Kokkos::parallel_for( - "Set energies", numParticles, KOKKOS_LAMBDA(int i) { - physics.set_energy(particles(i), energy); - physics.collide_particle(particles(i), fields(i)); - }); - - //Reset weight - Kokkos::parallel_for( - "initialize_particles", numParticles, KOKKOS_LAMBDA(int i) { - particles(i).weight = 1.0; - }); - - // Call the function to be tested - Kokkos::parallel_for( - "run physics", numParticles, KOKKOS_LAMBDA(int i) { - int j = 0; - while(particles(i).weight > 0.0001){ - //if (j != 0){ - // Kokkos::resize(track, (track.extent(0)+1)); - //} - physics.sample_collision_distance(particles(i), fields(i)); - physics.collide_particle(particles(i), fields(i)); - track(j) = particles(i); - j++; - } - }); - auto output = create_mirror_view(track); - Kokkos::deep_copy(output, track); - - const int length = output.extent(0); - - std::ofstream outfile("Log.txt"); - outfile << "Particle #,Energy(eV),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; - - for (int i = 0; i < length; ++i) { - outfile << i << ","; - outfile << output(i).energy_group << ","; - outfile << output(i).weight << ","; - outfile << output(i).position[0] << ","; - outfile << output(i).position[1] << ","; - outfile << output(i).position[2] << ","; - outfile << output(i).direction[0] << ","; - outfile << output(i).direction[1] << ","; - outfile << output(i).direction[2] << std::endl; - } - REQUIRE_THAT(output(length-1).weight, Catch::Matchers::WithinAbs(0,.03)); - - } - Kokkos::finalize(); - -} \ No newline at end of file From 7c4d3b3b18f101ea730d4c1985050a617f392f88 Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Mon, 14 Jul 2025 18:58:14 -0400 Subject: [PATCH 094/139] degas2 main added - addd binary but not installing yet - read parameter function - 6 element osh mesh for testing --- .gitignore | 1 + CMakeLists.txt | 10 +++++ src/physics/degas2/degas2-main.cpp | 59 +++++++++++++++++++++++++++++ test/assets/tet6-222.osh/0.osh | Bin 0 -> 1446 bytes test/assets/tet6-222.osh/nparts | 1 + test/assets/tet6-222.osh/version | 1 + 6 files changed, 72 insertions(+) create mode 100644 src/physics/degas2/degas2-main.cpp create mode 100644 test/assets/tet6-222.osh/0.osh create mode 100644 test/assets/tet6-222.osh/nparts create mode 100644 test/assets/tet6-222.osh/version diff --git a/.gitignore b/.gitignore index 27bd5df..471f394 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ build/ .idea/ config.sh Testing/ +*.vtk diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b35577..94b5e4e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,6 +96,16 @@ if(PUMI_ENABLE_DEGAS2) target_sources(pumitally_degas2 PRIVATE ${PUMI_DEGAS2_SOURCES}) target_link_libraries(pumitally_degas2 PUBLIC Omega_h::omega_h HDF5::HDF5) target_include_directories(pumitally_degas2 PUBLIC ${HDF5_INCLUDE_DIRS}) + + + add_executable(degas2 src/physics/degas2/degas2-main.cpp) + target_link_libraries(degas2 PRIVATE pumitally_degas2 pumitallyopenmc) + target_include_directories(degas2 + PRIVATE + $ + $ + ${CMAKE_SOURCE_DIR}/src/pumitallyopenmc + ) endif() ################################################################### diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp new file mode 100644 index 0000000..c67dff5 --- /dev/null +++ b/src/physics/degas2/degas2-main.cpp @@ -0,0 +1,59 @@ +/* + * This is created as a proof of concept for the Degas2 Neutral Transport Code + * Provides parallel search capabilities and tallying to degas2 through this + * PUMI-Tally interface + */ + +#include "DG2Physics.h" +#include + +// ******************************************* Helper Functions +// ******************************************* // +void read_input_parameters(int argc, char *const *argv, std::string &mesh_name, + int &num_particles); +void print_initial_info(const std::string &mesh_name, + int num_particles) noexcept; + +int main(int argc, char *argv[]) { + + std::string mesh_name; + int num_particles; + read_input_parameters(argc, argv, mesh_name, num_particles); + print_initial_info(mesh_name, num_particles); + + // Initialize PUMI-Tally + auto pumitally = + pumiinopenmc::PumiTally(mesh_name, num_particles, argc, argv); + + return 0; +} + +// **************************************************************************************************************** +// // +// ********************************************* Function Definitions +// ********************************************* // +// **************************************************************************************************************** +// // + +void print_initial_info(const std::string &mesh_name, + int num_particles) noexcept { + printf("\n===================> Accelerated Degas2 <======================\n"); + std::string mesh_particle_info = + "\tMesh: " + mesh_name + + "\n\tNumber of particles: " + std::to_string(num_particles) + "\n"; + printf("%s", mesh_particle_info.c_str()); + printf("\n===============================================================\n"); +} + +void read_input_parameters(int argc, char *const *argv, std::string &mesh_name, + int &num_particles) { + if (argc != 3) { + throw std::runtime_error("Usage: " + std::string(argv[0]) + + " "); + } + mesh_name = argv[1]; + num_particles = std::stoi(argv[2]); + if (num_particles <= 0) { + throw std::runtime_error("Number of particles must be a positive integer."); + } +} diff --git a/test/assets/tet6-222.osh/0.osh b/test/assets/tet6-222.osh/0.osh new file mode 100644 index 0000000000000000000000000000000000000000..085bb0704c5cc0d84360e9e68efecfb050f300f9 GIT binary patch literal 1446 zcmZ1|#mKwznl=ZVd{IxXx`({DCTx#p&vv-7ME&RtU@59hw1~eoBVbEE2E}%iq7faq({y!YX zf9ZhwrVUSmydn>Y+%jnS-z%7HaY%BO*Yx{(zY^NFUyo#uuj4Df@SZP!^@o&&(f?L$ zo^5jet@?+Oy^d|y*7vhZ?v`KR!T17Xha%jL*Qo$>0;2804g+vE7m>Z&c|TDam3+L_3!`b$trxx-L?u!mo7edxTaOjC3_jOFyrZW z?&c--JHq}(eq>Y-VYmn~Kn!j`aza`{Y63%YTABj$sWb)##(D-x7N9IQ5GUv77o}w8 zC6=TXGqZvO^5WA|^K;`<^Gb?A27pXb0WpA}f-yNkfzx8a%C?V*CO%T)^CwvN$hjFi zF!t~-PJG0}@caNXBO6d%dQN^)Vh$rS7UMwiCI>PL2tdX)Ft}-)Wm!6Np$YetwAM>G z0kdSim>HNJFvx;bCg&s;7sqF&Ff!pVN)~Jo#HfS>0j4BQg*4_QK80yaNrDP#Oi2vv z1`L^;aFbFpa~Z+DM)M@dE|3i%0C5!q0|x^mGmuNbKf*vI`XFsUP{F8s)RCz{fPujv zNRjLB|BbS@^td02@3pM_{_yh6qgSWT-konRH!rU4FQbYD!y1AfL^YfDFe6ukf&jw? z?%oYP-4|Z`m(Oip+MyBFKL76NH?N+(d|UGUW9i2YKbh4;7=94;GRSaIu(QElPDx2h zNl9T6I+GxDf{lSunn8vL&kF(d>%q;?J-VH-K>_G_7uL+C|MsU7HmS7SfB53^^nCSn zd%bme@8s%i_x`D5mb<|)i=f{@Mytb(PEJfnc)*wxr|`@vshja8lfxONrH-ErcqTEg zWc~Pafdqr12*Yc_eg+u|!XN;Q=A@)FCPjsVJc*19jB^>ZiSRHeF@Q1z2w+LQ42&KO z0tCGVGK3Fo1jvxbsX(tCZDM4JVelvH1CU;%gqp?%EEQcDgxP_@pbVXqpHc}bCqZ7} z1!5*3uFwXFfLI_)!6Lda5gw>WC0q?K@q*P Date: Mon, 14 Jul 2025 21:34:49 -0400 Subject: [PATCH 095/139] separated the impl header from the cpp file - tests are passing - openmc is compiling (pimpl structure is working) --- CMakeLists.txt | 2 + .../pumipic_particle_data_structure.cpp | 916 +----------------- .../pumipic_particle_data_structure.h | 4 +- src/pumitallyopenmc/pumitally_impl.cpp | 760 +++++++++++++++ src/pumitallyopenmc/pumitally_impl.hpp | 170 ++++ test/CMakeLists.txt | 2 +- test/test_pumi_tally_impl_methods.cpp | 5 +- 7 files changed, 937 insertions(+), 922 deletions(-) create mode 100644 src/pumitallyopenmc/pumitally_impl.cpp create mode 100644 src/pumitallyopenmc/pumitally_impl.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 94b5e4e..050a5d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,9 +28,11 @@ endif() ####################### Headers and Sources ####################### set(PUMITALLYOPENMC_HEADERS src/pumitallyopenmc/pumipic_particle_data_structure.h + src/pumitallyopenmc/pumitally_impl.hpp ) set(PUMITALLYOPENMC_SOURCES + src/pumitallyopenmc/pumitally_impl.cpp src/pumitallyopenmc/pumipic_particle_data_structure.cpp ) diff --git a/src/pumitallyopenmc/pumipic_particle_data_structure.cpp b/src/pumitallyopenmc/pumipic_particle_data_structure.cpp index ee82ebb..8b432b1 100644 --- a/src/pumitallyopenmc/pumipic_particle_data_structure.cpp +++ b/src/pumitallyopenmc/pumipic_particle_data_structure.cpp @@ -1,920 +1,7 @@ -// -// Created by Fuad Hasan on 12/3/24. -// - #include "pumipic_particle_data_structure.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include +#include "pumitally_impl.hpp" namespace pumiinopenmc { -struct TallyTimes { - double initialization_time = 0.0; - double total_time_to_tally = 0.0; - double vtk_file_write_time = 0.0; - - void print_times() const { - printf("\n"); - printf("[TIME] Initialization time : %f seconds\n", - initialization_time); - printf("[TIME] Total time to tally : %f seconds\n", - total_time_to_tally); - printf("[TIME] VTK file write time : %f seconds\n", - vtk_file_write_time); - printf("[TIME] Total PumiPic time : %f seconds\n", - initialization_time + total_time_to_tally + vtk_file_write_time); - } -}; - -// ------------------------------------------------------------------------------------------------// -// * Data structure for PumiPic -// Particle: 0-origin, 1-destination, 2-particle_id, -// 3-in_advance_particle_queue, 4-weight, 5-group -typedef pumipic::MemberTypes - PPParticle; -typedef pumipic::ParticleStructure PPPS; -typedef Kokkos::DefaultExecutionSpace PPExeSpace; - -// ------------------------------------------------------------------------------------------------// -// * Helper Functions -[[deprecated("Use move_to_next_element which is appropriate for new search " - "class in pumipic. [!Note] It my show this even though it is not " - "used due to template instantiation.")]] -void pp_move_to_new_element(Omega_h::Mesh &mesh, PPPS *ptcls, - Omega_h::Write &elem_ids, - Omega_h::Write &ptcl_done, - Omega_h::Write &lastExit); - -std::unique_ptr pp_create_particle_structure(Omega_h::Mesh mesh, - pumipic::lid_t numPtcls); - -void start_pumi_particles_in_0th_element(Omega_h::Mesh &mesh, - pumiinopenmc::PPPS *ptcls); - -PumiTally::~PumiTally() { - pimpl.reset(nullptr); - Kokkos::finalize(); -} - -// ------------------------------------------------------------------------------------------------// -// * Struct for PumiParticleAtElemBoundary -struct PumiParticleAtElemBoundary { - PumiParticleAtElemBoundary(size_t nelems, size_t capacity); - - void operator()(Omega_h::Mesh &mesh, pumiinopenmc::PPPS *ptcls, - Omega_h::Write &elem_ids, - Omega_h::Write &next_elems, - Omega_h::Write &inter_faces, - Omega_h::Write &lastExit, - Omega_h::Write &inter_points, - Omega_h::Write &ptcl_done, - typeof(ptcls->get<0>()) origin_segment, - typeof(ptcls->get<1>()) dest_segment); - - void updatePrevXPoint(Omega_h::Write &xpoints); - - void updatePrevXPoint(PPPS *ptcls); - - void evaluateFlux(PPPS *ptcls, Omega_h::Write xpoints, - Omega_h::Write elem_ids, - Omega_h::Write ptcl_done); - - void finalizeAndWritePumiFlux(Omega_h::Mesh &full_mesh, - const std::string &filename); - - void normalizeFlux(Omega_h::Mesh &mesh); - - void mark_initial_as(bool initial); - - void compute_total_tracklength(PPPS *ptcls); - void initialize_flux_array(size_t nelems, size_t nEgroups); - - bool initial_; // in initial run, flux is not tallied - // Dims: nelems, nEgroups, 3(flux, flux^2, standard deviation) - Kokkos::View flux_; - Omega_h::Write prev_xpoint_; - Omega_h::Write total_tracklength_; - Omega_h::Write material_ids_; -}; -// ------------------------------------------------------------------------------------------------// - -// ------------------------------------------------------------------------------------------------// -// * Struct for PumiTallyImpl -struct PumiTallyImpl { - int64_t pumi_ps_size = 1000000; // hundred thousand - std::string oh_mesh_fname; - - Omega_h::Library oh_lib; - Omega_h::Mesh full_mesh_; - - std::unique_ptr pp_lib = nullptr; - std::unique_ptr p_picparts_ = nullptr; - std::unique_ptr pumipic_ptcls = nullptr; - - long double pumipic_tol = 1e-8; - bool is_pumipic_initialized = false; - int64_t iter_count_ = 0; - double total_initial_weight_ = 0.0; - - std::unique_ptr - p_pumi_particle_at_elem_boundary_handler; - std::unique_ptr< - ParticleTracer> - p_particle_tracer_; - - Omega_h::Write device_pos_buffer_; - Omega_h::Write device_in_adv_que_; - Omega_h::Write weights_; - Omega_h::Write groups_; - - TallyTimes tally_times; - - // * Constructor - PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, int &argc, - char **&argv); // fixme extra & - - // * Destructor - ~PumiTallyImpl() = default; - - // Functions - void create_and_initialize_pumi_particle_structure(Omega_h::Mesh *mesh); - - void load_pumipic_mesh_and_init_particles(int &argc, char **&argv); - - Omega_h::Mesh *partition_pumipic_mesh(); - - void init_pumi_libs(int &argc, char **&argv); - - void search_and_rebuild(bool initial, bool migrate = true); - - void read_pumipic_lib_and_full_mesh(int &argc, char **&argv); - - void initialize_particle_location(double *init_particle_positions, - int64_t size); - - void move_to_next_location(double *particle_destinations, int8_t *flying, - double *weights, int *groups, int *material_ids, - int64_t size); - - void write_pumi_tally_mesh(); - - void copy_data_to_device(double *init_particle_positions); - - void search_initial_elements(); - - void copy_and_reset_flying_flag(int8_t *flying); - - void copy_weights(double *weights); - - void copy_groups(int *groups); - - void copy_last_location(double *particle_destination, int64_t size); - - void copy_material_ids(int *material_ids, int64_t size); -}; - -PumiTallyImpl::PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, - int &argc, char **&argv) { - pumi_ps_size = num_particles; - oh_mesh_fname = mesh_filename; - - device_pos_buffer_ = - Omega_h::Write(pumi_ps_size * 3, 0.0, "device_pos_buffer"); - device_in_adv_que_ = - Omega_h::Write(pumi_ps_size, 0, "device_in_adv_que"); - weights_ = Omega_h::Write(pumi_ps_size, 0.0, "weights"); - groups_ = Omega_h::Write(pumi_ps_size, 0, "groups"); - - // todo can track lengths be here? - - load_pumipic_mesh_and_init_particles(argc, argv); - start_pumi_particles_in_0th_element(*p_picparts_->mesh(), - pumipic_ptcls.get()); - - p_particle_tracer_ = std::make_unique< - ParticleTracer>( - *p_picparts_, pumipic_ptcls.get(), - *p_pumi_particle_at_elem_boundary_handler, 1e-8); -} - -void PumiTallyImpl::initialize_particle_location( - double *init_particle_positions, int64_t size) { - // copy to host buffer - assert(size == pumi_ps_size * 3); - copy_data_to_device(init_particle_positions); - search_initial_elements(); - // TODO Get total initial particle weight -#ifdef PUMI_MEASURE_TIME - Kokkos::fence(); -#endif -} - -void PumiTallyImpl::move_to_next_location(double *particle_destinations, - int8_t *flying, double *weights, - int *groups, int *material_ids, - int64_t size) { - assert(size == pumi_ps_size * 3); - - // copy to device buffer - copy_data_to_device(particle_destinations); - // copy fly to device buffer - copy_and_reset_flying_flag(flying); - copy_weights(weights); - copy_groups(groups); - - // copy position buffer ps - auto particle_dest = pumipic_ptcls->get<1>(); - auto in_flight = pumipic_ptcls->get<3>(); - - int64_t pumi_ps_size_ = pumi_ps_size; - const auto &device_pos_buffer_l = device_pos_buffer_; - const auto &device_in_adv_que_l = device_in_adv_que_; - - auto set_particle_dest = - PS_LAMBDA(const int &e, const int &pid, const int &mask) { - if (mask > 0 && pid < pumi_ps_size_) { - particle_dest(pid, 0) = device_pos_buffer_l[pid * 3 + 0]; - particle_dest(pid, 1) = device_pos_buffer_l[pid * 3 + 1]; - particle_dest(pid, 2) = device_pos_buffer_l[pid * 3 + 2]; - - // everyone is in flight for this initial search - in_flight(pid) = device_in_adv_que_l[pid]; - } - }; - pumipic::parallel_for(pumipic_ptcls.get(), set_particle_dest, - "set particle position as dest"); - - bool migrate = iter_count_ % 100 == 0; - iter_count_++; - search_and_rebuild(false, migrate); -#ifdef PUMI_MEASURE_TIME - Kokkos::fence(); -#endif - copy_last_location(particle_destinations, size); - copy_material_ids(material_ids, size / 3); -} - -void PumiTallyImpl::copy_last_location(double *particle_destination, - int64_t size) { - auto last_location = p_pumi_particle_at_elem_boundary_handler->prev_xpoint_; - OMEGA_H_CHECK_PRINTF(last_location.size() >= size, - "last_location size %d is not greater to %ld\n", - last_location.size(), size); - Kokkos::View> - device_last_location_view(last_location.data(), size); - Kokkos::View> - host_last_location_view(particle_destination, size); - - Kokkos::deep_copy(host_last_location_view, device_last_location_view); -} - -void PumiTallyImpl::copy_material_ids(int *material_ids, int64_t size) { - auto material_ids_l = p_pumi_particle_at_elem_boundary_handler->material_ids_; - OMEGA_H_CHECK_PRINTF(material_ids_l.size() >= size, - "material_ids_l size %d is not greater to %ld\n", - material_ids_l.size(), size); - Kokkos::View> - host_material_ids_view(material_ids, size); - Kokkos::View> - device_material_ids_view(material_ids_l.data(), size); - - Kokkos::deep_copy(host_material_ids_view, device_material_ids_view); -} - -void PumiTallyImpl::write_pumi_tally_mesh() { - p_pumi_particle_at_elem_boundary_handler->finalizeAndWritePumiFlux( - full_mesh_, "fluxresult.vtk"); -#ifdef PUMI_MEASURE_TIME - Kokkos::fence(); -#endif -} - -void PumiTallyImpl::copy_and_reset_flying_flag(int8_t *flying) { - // todo get the size too - auto device_in_adv_que_l = device_in_adv_que_; - Kokkos::View> - host_flying_view(flying, pumi_ps_size); - Kokkos::View> - device_flying_view(device_in_adv_que_l.data(), - device_in_adv_que_l.size()); - Kokkos::deep_copy(device_flying_view, host_flying_view); - - for (int64_t pid = 0; pid < pumi_ps_size; ++pid) { - // reset flying flag to zero - flying[pid] = 0; - } -} - -void PumiTallyImpl::copy_weights(double *weights) { - auto weights_l = weights_; - Kokkos::View> - host_weights_view(weights, pumi_ps_size); - Kokkos::View> - device_weights_view(weights_l.data(), weights_l.size()); - - Kokkos::deep_copy(device_weights_view, host_weights_view); - - auto p_wgt = pumipic_ptcls->get<4>(); - auto copy_particle_weights = - PS_LAMBDA(const int &e, const int &pid, const int &mask) { - p_wgt(pid) = weights_l[pid]; - }; - pumipic::parallel_for(pumipic_ptcls.get(), copy_particle_weights, - "copy particle weights"); -} - -void PumiTallyImpl::copy_groups(int *groups) { - auto groups_l = groups_; - Kokkos::View> - host_groups_view(groups, pumi_ps_size); - Kokkos::View> - device_groups_view(groups_l.data(), groups_l.size()); - - Kokkos::deep_copy(device_groups_view, host_groups_view); - auto p_groups = pumipic_ptcls->get<5>(); - auto copy_particle_groups = - PS_LAMBDA(const int &e, const int &pid, const int &mask) { - p_groups(pid) = groups_l[pid]; - }; - pumipic::parallel_for(pumipic_ptcls.get(), copy_particle_groups, - "copy particle groups"); -} - -void PumiTallyImpl::search_initial_elements() { // assign the location to ptcl - // dest - auto particle_dest = pumipic_ptcls->get<1>(); - auto in_flight = pumipic_ptcls->get<3>(); - - int64_t pumi_ps_size_l = pumi_ps_size; - const auto &device_pos_buffer_l = device_pos_buffer_; - - auto set_particle_dest = - PS_LAMBDA(const int &e, const int &pid, const int &mask) { - if (mask > 0 && pid < pumi_ps_size_l) { - particle_dest(pid, 0) = device_pos_buffer_l[pid * 3 + 0]; - particle_dest(pid, 1) = device_pos_buffer_l[pid * 3 + 1]; - particle_dest(pid, 2) = device_pos_buffer_l[pid * 3 + 2]; - - // everyone is in flight for this initial search - in_flight(pid) = 1; - } - }; - pumipic::parallel_for(pumipic_ptcls.get(), set_particle_dest, - "set initial position as dest"); - - // *initial* build and search to find the initial elements of the particles - search_and_rebuild(true, true); - is_pumipic_initialized = true; -} - -void PumiTallyImpl::copy_data_to_device(double *init_particle_positions) { - // fixme it should get size too to avoid memory error - Kokkos::View> - host_pos_view(init_particle_positions, pumi_ps_size * 3); - - Kokkos::View> - device_pos_view(device_pos_buffer_.data(), pumi_ps_size * 3); - - Kokkos::deep_copy(device_pos_view, host_pos_view); -} - -// methods for PumiTallyImpl and PumiParticleAtElemBoundary - -void PumiTallyImpl::init_pumi_libs(int &argc, char **&argv) { - pp_lib = std::make_unique(&argc, &argv); - oh_lib = pp_lib->omega_h_lib(); -} - -[[deprecated("Use move_to_next_element which is appropriate for new search " - "class in pumipic. [!Note] It my show this even though it is not " - "used due to template instantiation.")]] -void pp_move_to_new_element(Omega_h::Mesh &mesh, PPPS *ptcls, - Omega_h::Write &elem_ids, - Omega_h::Write &ptcl_done, - Omega_h::Write &lastExit) { - const int dim = mesh.dim(); - const auto &face2elems = mesh.ask_up(dim - 1, dim); - const auto &face2elemElem = face2elems.ab2b; - const auto &face2elemOffset = face2elems.a2ab; - const auto in_flight = ptcls->get<3>(); - - auto set_next_element = - PS_LAMBDA(const int &e, const int &pid, const int &mask) { - if (mask > 0 && !ptcl_done[pid] && in_flight(pid)) { - auto searchElm = elem_ids[pid]; - auto bridge = lastExit[pid]; - auto e2f_first = face2elemOffset[bridge]; - auto e2f_last = face2elemOffset[bridge + 1]; - auto upFaces = e2f_last - e2f_first; - assert(upFaces == 2); - auto faceA = face2elemElem[e2f_first]; - auto faceB = face2elemElem[e2f_first + 1]; - assert(faceA != faceB); - assert(faceA == searchElm || faceB == searchElm); - auto nextElm = (faceA == searchElm) ? faceB : faceA; - elem_ids[pid] = nextElm; - } - }; - parallel_for(ptcls, set_next_element, "pumipic_set_next_element"); -} - -void move_to_next_element(PPPS *ptcls, Omega_h::Write &elem_ids, - Omega_h::Write &next_elems) { - auto in_flight = ptcls->get<3>(); - auto move_to_next = PS_LAMBDA(const int e, const int pid, const int mask) { - // move only if particle in flight and not leaving the domain - if (mask > 0 && in_flight(pid) && next_elems[pid] != -1) { - elem_ids[pid] = next_elems[pid]; - } - }; - pumipic::parallel_for(ptcls, move_to_next, "move to next element"); -} - -void apply_boundary_condition(Omega_h::Mesh &mesh, PPPS *ptcls, - Omega_h::Write &elem_ids, - Omega_h::Write &next_elems, - Omega_h::Write &ptcl_done, - Omega_h::Write &lastExit, - Omega_h::Write &xFace, - Omega_h::Write &inter_points, - Omega_h::Write material_ids, bool initial) { - - // TODO: make this a member variable of the struct - auto particle_destination = ptcls->get<1>(); - const auto class_ids = mesh.get_array(3, "class_id"); - auto checkExposedEdges = - PS_LAMBDA(const int e, const int pid, const int mask) { - if (mask > 0 && !ptcl_done[pid]) { - bool reached_destination = (lastExit[pid] == -1); - bool hit_boundary = ((next_elems[pid] == -1) && (elem_ids[pid] != -1)); - - // for the initial run, we need to find the initial position of the - // particles - if (!initial) { // stop at geometry boundary - if (next_elems[pid] != -1) { - if (class_ids[elem_ids[pid]] != - class_ids[next_elems[pid]]) { // particle crosses geometry - // boundary - hit_boundary = true; - material_ids[pid] = class_ids[next_elems[pid]]; - } - } else { - material_ids[pid] = -1; // no material id if not in an element - } - } - - ptcl_done[pid] = - (reached_destination || hit_boundary) ? 1 : ptcl_done[pid]; - // assert that if the next element is -1, then the material id is -1 - if (!initial) { - if (next_elems[pid] == -1) { - OMEGA_H_CHECK_PRINTF( - material_ids[pid] == -1, - "Error: next_elems[%d] is -1 but material_ids[%d] " - "is %d\n", - pid, pid, material_ids[pid]); - } - // printf("Pid %d next element %d, elem id %d, material id %d\n", - // pid, next_elems[pid], elem_ids[pid], material_ids[pid]); - } - - if (hit_boundary) { // just reached the boundary - // printf("Moving particle %4d (from -> to): element (%5d -> %5d) - // Material (%3d -> %3d)\n", - // pid, elem_ids[pid], next_elems[pid], - // class_ids[elem_ids[pid]], material_ids[pid]); - xFace[pid] = lastExit[pid]; - // particle reaches the boundary - particle_destination(pid, 0) = inter_points[pid * 3]; - particle_destination(pid, 1) = inter_points[pid * 3 + 1]; - particle_destination(pid, 2) = inter_points[pid * 3 + 2]; - } - } - }; - pumipic::parallel_for(ptcls, checkExposedEdges, - "apply vacumm boundary condition"); -} - -void PumiParticleAtElemBoundary::initialize_flux_array(size_t nelems, - size_t nEgroups) { - Kokkos::resize(flux_, nelems, nEgroups, 3); - auto flux_l = flux_; - OMEGA_H_CHECK(flux_l.extent(0) == nelems); - OMEGA_H_CHECK(flux_l.extent(1) == nEgroups); - OMEGA_H_CHECK(flux_l.extent(2) == 3); -} - -PumiParticleAtElemBoundary::PumiParticleAtElemBoundary(size_t nelems, - size_t capacity) - : prev_xpoint_(capacity * 3, 0.0, "prev_xpoint"), - material_ids_(capacity, -1, "material_ids"), - total_tracklength_(capacity, 0.0, "total_tracklength"), initial_(true) { - initialize_flux_array(nelems, 2); // FIXME: hardcoded 2 groups - printf("[INFO] Particle handler at boundary with %d elements and %d " - "x points size (3 * n_particles)\n", - flux_.size(), prev_xpoint_.size()); -} - -void PumiParticleAtElemBoundary::operator()( - Omega_h::Mesh &mesh, pumiinopenmc::PPPS *ptcls, - Omega_h::Write &elem_ids, - Omega_h::Write &next_elems, - Omega_h::Write &inter_faces, - Omega_h::Write &lastExit, - Omega_h::Write &inter_points, - Omega_h::Write &ptcl_done, - typeof(ptcls->get<0>()) origin_segment, - typeof(ptcls->get<1>()) dest_segment) { - if (!initial_) { - evaluateFlux(ptcls, inter_points, elem_ids, ptcl_done); - updatePrevXPoint(inter_points); - } - apply_boundary_condition(mesh, ptcls, elem_ids, next_elems, ptcl_done, - lastExit, inter_faces, inter_points, material_ids_, - initial_); - move_to_next_element(ptcls, elem_ids, next_elems); -} - -void PumiParticleAtElemBoundary::mark_initial_as(bool initial) { - initial_ = initial; -} - -void PumiParticleAtElemBoundary::updatePrevXPoint( - Omega_h::Write &xpoints) { - OMEGA_H_CHECK_PRINTF(xpoints.size() <= prev_xpoint_.size() && - prev_xpoint_.size() != 0, - "xpoints size %d is greater than prev_xpoint size %d\n", - xpoints.size(), prev_xpoint_.size()); - auto &prev_xpoint = prev_xpoint_; - auto update = OMEGA_H_LAMBDA(Omega_h::LO i) { prev_xpoint[i] = xpoints[i]; }; - Omega_h::parallel_for(xpoints.size(), update, "update previous xpoints"); -} - -void PumiParticleAtElemBoundary::updatePrevXPoint(PPPS *ptcls) { - // todo add checks of size - auto prev_xpoints_l = prev_xpoint_; - OMEGA_H_CHECK_PRINTF( - ptcls->capacity() * 3 == prev_xpoints_l.size(), - "Error: prev_xpoints_s are not size properly capacity %d size %d\n", - ptcls->capacity(), prev_xpoints_l.size()); - auto xpoints = ptcls->get<0>(); - auto update = PS_LAMBDA(const auto &e, const auto &pid, const auto &mask) { - prev_xpoints_l[pid * 3 + 0] = xpoints(pid, 0); - prev_xpoints_l[pid * 3 + 1] = xpoints(pid, 1); - prev_xpoints_l[pid * 3 + 2] = xpoints(pid, 2); - }; - pumipic::parallel_for(ptcls, update, - "update previous xpoints from origin points"); -} - -void PumiParticleAtElemBoundary::evaluateFlux( - PPPS *ptcls, Omega_h::Write xpoints, - Omega_h::Write elem_ids, - Omega_h::Write ptcl_done) { - // Omega_h::Real total_particles = ptcls->nPtcls(); - auto prev_xpoint = prev_xpoint_; - auto flux = flux_; - auto total_tracklength_l = total_tracklength_; - auto in_flight = ptcls->get<3>(); - auto p_wgt = ptcls->get<4>(); - auto p_groups = ptcls->get<5>(); - auto xpoints_l = xpoints; // todo shouldn't need it, so remove - - auto evaluate_flux = - PS_LAMBDA(const int &e, const int &pid, const int &mask) { - if ((mask > 0) && (in_flight(pid) == 1) && !ptcl_done[pid]) { - OMEGA_H_CHECK_PRINTF( - total_tracklength_l[pid] >= 0.0, - "ERROR: Particle is moving but the tracklength is negative: %.16f\n", - total_tracklength_l[pid]); - - Omega_h::Vector<3> dest = {xpoints_l[pid * 3 + 0], xpoints_l[pid * 3 + 1], - xpoints_l[pid * 3 + 2]}; - Omega_h::Vector<3> orig = {prev_xpoint[pid * 3 + 0], - prev_xpoint[pid * 3 + 1], - prev_xpoint[pid * 3 + 2]}; - - Omega_h::Real segment_length = - Omega_h::norm(dest - orig); // / total_particles; - if (segment_length > - total_tracklength_l[pid] + 1e-6) { // tol for float operations and - // search algorithm's inaccuracy - // fixme: something wrong with orig: previous_xpoints are incorrect at - // least for the first run - printf("ERROR: Segment length in an element cannot be greater than the " - "total tracklength but found %.16f, %.16f of pid %d crossing el " - "%d starting at %d\nOrig: (%.16f, %.16f, %.16f), Dest: (%.16f, " - "%.16f, %.16f)\n", - segment_length, total_tracklength_l[pid], pid, elem_ids[pid], e, - orig[0], orig[1], orig[2], dest[0], dest[1], dest[2]); - } - - Omega_h::Real contribution = segment_length * p_wgt(pid); - - int group = p_groups(pid); - OMEGA_H_CHECK_PRINTF( - group >= 0 && group < flux.extent(1), - "ERROR: Group %d is out of bounds for flux array with " - "extent %lu\n", - group, flux.extent(1)); - - Kokkos::atomic_add(&flux(elem_ids[pid], group, 0), contribution); - Kokkos::atomic_add(&flux(elem_ids[pid], group, 1), - contribution * contribution); - } - }; - pumipic::parallel_for(ptcls, evaluate_flux, "flux evaluation loop"); -} - -void PumiParticleAtElemBoundary::normalizeFlux(Omega_h::Mesh &mesh) { - const Omega_h::LO nelems = mesh.nelems(); - const auto &el2n = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; - const auto &coords = mesh.coords(); - - auto flux = flux_; - auto total_tracklength_l = total_tracklength_; - - Omega_h::Write tet_volumes(flux_.extent(0), -1.0, - "tet_volumes"); - Omega_h::Write sd(flux_.extent(0), -1.0, "sd"); - - auto normalize_flux_with_volume = OMEGA_H_LAMBDA(Omega_h::LO elem_id) { - const auto elem_verts = Omega_h::gather_verts<4>(el2n, elem_id); - const auto elem_vert_coords = - Omega_h::gather_vectors<4, 3>(coords, elem_verts); - - auto b = Omega_h::simplex_basis<3, 3>(elem_vert_coords); - auto volume = Omega_h::simplex_size_from_basis(b); - - tet_volumes[elem_id] = volume; - for (int g = 0; g < flux.extent(1); g++) { - flux(elem_id, g, 0) /= (volume * total_tracklength_l.size()); - flux(elem_id, g, 1) /= (volume * volume * total_tracklength_l.size()); - - // calculate standard deviation FIXME this is not correct, needs number of - // iterations - flux(elem_id, g, 2) = Kokkos::sqrt( - flux(elem_id, g, 1) - flux(elem_id, g, 0) * flux(elem_id, g, 0)); - } - }; - Omega_h::parallel_for(tet_volumes.size(), normalize_flux_with_volume, - "normalize flux"); - - mesh.add_tag(Omega_h::REGION, "volume", 1, Omega_h::Reals(tet_volumes)); -} - -void PumiParticleAtElemBoundary::finalizeAndWritePumiFlux( - Omega_h::Mesh &full_mesh, const std::string &filename) { - normalizeFlux(full_mesh); - auto flux = flux_; - int num_groups = flux.extent(1); - int num_elems = flux.extent(0); - Omega_h::Write normalized_flux(num_elems, 0.0, - "normalized_flux"); - // copy flux to normalized_flux - for (int g = 0; g < num_groups; g++) { - auto copy_flux = OMEGA_H_LAMBDA(Omega_h::LO elem_id) { - normalized_flux[elem_id] = flux(elem_id, g, 0); - }; - Omega_h::parallel_for(num_elems, copy_flux, "copy flux to normalized_flux"); - std::string tag_name = "flux_group_" + std::to_string(g); - Kokkos::fence(); - full_mesh.add_tag(Omega_h::REGION, tag_name, 1, - Omega_h::Reals(normalized_flux)); - } - Omega_h::vtk::write_parallel(filename, &full_mesh, 3); -} - -void pumiUpdatePtclPositions(PPPS *ptcls) { - auto x_ps_d = ptcls->get<0>(); - auto xtgt_ps_d = ptcls->get<1>(); - auto updatePtclPos = PS_LAMBDA(const int &, const int &pid, const bool &) { - x_ps_d(pid, 0) = xtgt_ps_d(pid, 0); - x_ps_d(pid, 1) = xtgt_ps_d(pid, 1); - x_ps_d(pid, 2) = xtgt_ps_d(pid, 2); - xtgt_ps_d(pid, 0) = 0.0; - xtgt_ps_d(pid, 1) = 0.0; - xtgt_ps_d(pid, 2) = 0.0; - }; - ps::parallel_for(ptcls, updatePtclPos); -} - -void PumiParticleAtElemBoundary::compute_total_tracklength(PPPS *ptcls) { - auto orig = ptcls->get<0>(); - auto dest = ptcls->get<1>(); - - auto total_tracklength_l = total_tracklength_; - - auto computeTrackLength = - PS_LAMBDA(const int &elemId, const int &pid, const bool &mask) { - Omega_h::Vector<3> p_orig = {orig(pid, 0), orig(pid, 1), orig(pid, 2)}; - Omega_h::Vector<3> p_dest = {dest(pid, 0), dest(pid, 1), dest(pid, 2)}; - Omega_h::Real track_length = Omega_h::norm(p_dest - p_orig); - total_tracklength_l[pid] = track_length; - }; - pumipic::parallel_for(ptcls, computeTrackLength, - "compute total track length"); -} - -// search and update parent elements -//! @param initial initial search finds the initial location of the particles -//! and doesn't tally -void PumiTallyImpl::search_and_rebuild(bool initial, const bool migrate) { - // initial cannot be false when is_pumipic_initialized is false - // may fail if simulated more than one batch - assert((is_pumipic_initialized == false && initial == true) || - (is_pumipic_initialized == true && initial == false)); - p_pumi_particle_at_elem_boundary_handler->mark_initial_as(initial); - auto orig = pumipic_ptcls->get<0>(); - auto dest = pumipic_ptcls->get<1>(); - auto pid = pumipic_ptcls->get<2>(); - - if (p_picparts_->mesh() == nullptr || p_picparts_->mesh()->nelems() == 0) { - fprintf(stderr, "ERROR: Mesh is empty\n"); - } - - // total tracklengths are used to calculate the flux - if (!initial) { - p_pumi_particle_at_elem_boundary_handler->updatePrevXPoint( - pumipic_ptcls.get()); - p_pumi_particle_at_elem_boundary_handler->compute_total_tracklength( - pumipic_ptcls.get()); - } - - bool isFoundAll = p_particle_tracer_->search(migrate); - - if (!isFoundAll) { - printf( - "ERROR: Not all particles are found. May need more loops in search\n"); - } -} - -std::unique_ptr pp_create_particle_structure(Omega_h::Mesh mesh, - pumipic::lid_t numPtcls) { - Omega_h::Int ne = mesh.nelems(); - pumiinopenmc::PPPS::kkLidView ptcls_per_elem("ptcls_per_elem", ne); - pumiinopenmc::PPPS::kkGidView element_gids("element_gids", ne); - - Kokkos::TeamPolicy policy; - - Omega_h::parallel_for( - ne, OMEGA_H_LAMBDA(const Omega_h::LO &i) { element_gids(i) = i; }); - - Omega_h::parallel_for( - mesh.nelems(), OMEGA_H_LAMBDA(Omega_h::LO id) { - ptcls_per_elem[id] = (id == 0) ? numPtcls : 0; - }); - -#ifdef PUMI_USE_KOKKOS_CUDA - printf("PumiPIC Using GPU for simulation...\n"); - policy = Kokkos::TeamPolicy(10000, 32); -#else - printf("PumiPIC Using CPU for simulation...\n"); - policy = - Kokkos::TeamPolicy(10000, Kokkos::AUTO()); -#endif - - auto ptcls = std::make_unique>( - policy, ne, numPtcls, ptcls_per_elem, element_gids); - - return ptcls; -} - -/* -void set_pumipic_particle_structure_size(int openmc_particles_in_flight, int -openmc_work_per_rank, int openmc_n_particles) -{ - int64_t n_particles; // TODO have a better way to do it than this - // FIXME why this work_per rank is not set in the settings by now? - - - if (openmc_particles_in_flight == 0 && openmc_work_per_rank == 0) { - printf("While creating PumiPIC particle structure, both -max_particles_in_flight and work_per_rank are 0.\n"); n_particles = -(openmc_n_particles != 0) ? openmc_n_particles : pumi_ps_size; }else if -(openmc_particles_in_flight == 0 || openmc_work_per_rank == 0) { n_particles = -std::max(openmc_particles_in_flight, openmc_work_per_rank); printf("One of -max_particles_in_flight or work_per_rank is 0. Setting PumiPIC particle -structure size to %d\n", n_particles); } else { n_particles = -std::min(openmc_particles_in_flight, openmc_work_per_rank); printf("Setting -PumiPIC particle structure size to %d\n", n_particles); - } - - pumi_ps_size = n_particles; - printf("Creteating PumiPIC particle structure with size %d\n", n_particles); -} -*/ - -void start_pumi_particles_in_0th_element(Omega_h::Mesh &mesh, - pumiinopenmc::PPPS *ptcls) { - // find the centroid of the 0th element - const auto &coords = mesh.coords(); - const auto &tet2node = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; - - Omega_h::Write centroid_of_el0(3, 0.0, "centroid"); - - auto find_centroid_of_el0 = OMEGA_H_LAMBDA(Omega_h::LO id) { - const auto nodes = Omega_h::gather_verts<4>(tet2node, id); - Omega_h::Few, 4> tet_node_coords = - Omega_h::gather_vectors<4, 3>(coords, nodes); - const auto centroid = o::average(tet_node_coords); - centroid_of_el0[0] = centroid[0]; - centroid_of_el0[1] = centroid[1]; - centroid_of_el0[2] = centroid[2]; - }; - Omega_h::parallel_for(1, find_centroid_of_el0, "find centroid of element 0"); - - // assign the location to all particles - auto init_loc = ptcls->get<0>(); - auto pids = ptcls->get<2>(); - auto in_fly = ptcls->get<3>(); - - auto set_initial_positions = - PS_LAMBDA(const int &e, const int &pid, const int &mask) { - if (mask > 0) { - pids(pid) = pid; - in_fly(pid) = 1; - init_loc(pid, 0) = centroid_of_el0[0]; - init_loc(pid, 1) = centroid_of_el0[1]; - init_loc(pid, 2) = centroid_of_el0[2]; - } - }; - pumipic::parallel_for(ptcls, set_initial_positions, - "set initial particle positions"); -} - -Omega_h::Mesh *PumiTallyImpl::partition_pumipic_mesh() { - Omega_h::Write owners(full_mesh_.nelems(), 0, "owners"); - // all the particles are initialized in element 0 to do an initial search to - // find the starting locations - // of the openmc given particles. - // p_picparts_ = new pumipic::Mesh(full_mesh_, Omega_h::LOs(owners)); - p_picparts_ = - std::make_unique(full_mesh_, Omega_h::LOs(owners)); - printf("PumiPIC mesh partitioned\n"); - Omega_h::Mesh *mesh = p_picparts_->mesh(); - return mesh; -} - -void PumiTallyImpl::create_and_initialize_pumi_particle_structure( - Omega_h::Mesh *mesh) { - pumipic_ptcls = pp_create_particle_structure(*mesh, pumi_ps_size); - start_pumi_particles_in_0th_element(*mesh, pumipic_ptcls.get()); - p_pumi_particle_at_elem_boundary_handler = - std::make_unique( - mesh->nelems(), pumipic_ptcls->capacity()); - - printf("PumiPIC Mesh and data structure created with %d and %d as particle " - "structure capacity\n", - p_picparts_->mesh()->nelems(), pumipic_ptcls->capacity()); -} - -void PumiTallyImpl::read_pumipic_lib_and_full_mesh(int &argc, char **&argv) { - printf("Reading the Omega_h mesh %s to tally with tracklength estimator\n", - oh_mesh_fname.c_str()); - init_pumi_libs(argc, argv); - - if (oh_mesh_fname.empty()) { - printf("[ERROR] Omega_h mesh for PumiPIC is not given. Provide --ohMesh = " - ""); - } - full_mesh_ = Omega_h::binary::read(oh_mesh_fname, &oh_lib); - if (full_mesh_.dim() != 3) { - printf("PumiPIC only works for 3D mesh now.\n"); - } - OMEGA_H_CHECK_PRINTF(full_mesh_.has_tag(Omega_h::REGION, "class_id"), - "Mesh %s does not have class_id tag on regions.\n", - oh_mesh_fname.c_str()); - printf("PumiPIC Loaded mesh %s with %d elements\n", oh_mesh_fname.c_str(), - full_mesh_.nelems()); -} - -void PumiTallyImpl::load_pumipic_mesh_and_init_particles(int &argc, - char **&argv) { - read_pumipic_lib_and_full_mesh(argc, argv); - Omega_h::Mesh *mesh = partition_pumipic_mesh(); - create_and_initialize_pumi_particle_structure(mesh); -} - PumiTally::PumiTally(std::string &mesh_filename, int64_t num_particles, int &argc, char **&argv) : pimpl(std::make_unique(mesh_filename, num_particles, argc, @@ -955,5 +42,4 @@ void PumiTally::write_pumi_tally_mesh() { pimpl->tally_times.vtk_file_write_time += elapsed_seconds.count(); pimpl->tally_times.print_times(); } - } // namespace pumiinopenmc diff --git a/src/pumitallyopenmc/pumipic_particle_data_structure.h b/src/pumitallyopenmc/pumipic_particle_data_structure.h index e5cd6b5..8306962 100644 --- a/src/pumitallyopenmc/pumipic_particle_data_structure.h +++ b/src/pumitallyopenmc/pumipic_particle_data_structure.h @@ -11,8 +11,8 @@ namespace pumiinopenmc { // ------------------------------------------------------------------------------------------------// // Struct for PumiTallyImpl -struct PumiParticleAtElemBoundary; -struct PumiTallyImpl; +class PumiParticleAtElemBoundary; +class PumiTallyImpl; // PumiTally class // This is a pimpl class that contains the implementation of the PumiTallyImpl diff --git a/src/pumitallyopenmc/pumitally_impl.cpp b/src/pumitallyopenmc/pumitally_impl.cpp new file mode 100644 index 0000000..97ec319 --- /dev/null +++ b/src/pumitallyopenmc/pumitally_impl.cpp @@ -0,0 +1,760 @@ +// +// Created by Fuad Hasan on 12/3/24. +// + +#include "pumitally_impl.hpp" +#include "pumipic_particle_data_structure.h" +#include +#include +#include +#include + +namespace pumiinopenmc { + +void TallyTimes::print_times() const { + printf("\n"); + printf("[TIME] Initialization time : %f seconds\n", initialization_time); + printf("[TIME] Total time to tally : %f seconds\n", total_time_to_tally); + printf("[TIME] VTK file write time : %f seconds\n", vtk_file_write_time); + printf("[TIME] Total PumiPic time : %f seconds\n", + initialization_time + total_time_to_tally + vtk_file_write_time); +} +PumiTally::~PumiTally() { + pimpl.reset(nullptr); + Kokkos::finalize(); +} + +PumiTallyImpl::PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, + int &argc, char **&argv) { + pumi_ps_size = num_particles; + oh_mesh_fname = mesh_filename; + + device_pos_buffer_ = + Omega_h::Write(pumi_ps_size * 3, 0.0, "device_pos_buffer"); + device_in_adv_que_ = + Omega_h::Write(pumi_ps_size, 0, "device_in_adv_que"); + weights_ = Omega_h::Write(pumi_ps_size, 0.0, "weights"); + groups_ = Omega_h::Write(pumi_ps_size, 0, "groups"); + + // todo can track lengths be here? + + load_pumipic_mesh_and_init_particles(argc, argv); + start_pumi_particles_in_0th_element(*p_picparts_->mesh(), + pumipic_ptcls.get()); + + p_particle_tracer_ = std::make_unique< + ParticleTracer>( + *p_picparts_, pumipic_ptcls.get(), + *p_pumi_particle_at_elem_boundary_handler, 1e-8); +} + +void PumiTallyImpl::initialize_particle_location( + double *init_particle_positions, int64_t size) { + // copy to host buffer + assert(size == pumi_ps_size * 3); + copy_data_to_device(init_particle_positions); + search_initial_elements(); + // TODO Get total initial particle weight +#ifdef PUMI_MEASURE_TIME + Kokkos::fence(); +#endif +} + +void PumiTallyImpl::move_to_next_location(double *particle_destinations, + int8_t *flying, double *weights, + int *groups, int *material_ids, + int64_t size) { + assert(size == pumi_ps_size * 3); + + // copy to device buffer + copy_data_to_device(particle_destinations); + // copy fly to device buffer + copy_and_reset_flying_flag(flying); + copy_weights(weights); + copy_groups(groups); + + // copy position buffer ps + auto particle_dest = pumipic_ptcls->get<1>(); + auto in_flight = pumipic_ptcls->get<3>(); + + int64_t pumi_ps_size_ = pumi_ps_size; + const auto &device_pos_buffer_l = device_pos_buffer_; + const auto &device_in_adv_que_l = device_in_adv_que_; + + auto set_particle_dest = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if (mask > 0 && pid < pumi_ps_size_) { + particle_dest(pid, 0) = device_pos_buffer_l[pid * 3 + 0]; + particle_dest(pid, 1) = device_pos_buffer_l[pid * 3 + 1]; + particle_dest(pid, 2) = device_pos_buffer_l[pid * 3 + 2]; + + // everyone is in flight for this initial search + in_flight(pid) = device_in_adv_que_l[pid]; + } + }; + pumipic::parallel_for(pumipic_ptcls.get(), set_particle_dest, + "set particle position as dest"); + + bool migrate = iter_count_ % 100 == 0; + iter_count_++; + search_and_rebuild(false, migrate); +#ifdef PUMI_MEASURE_TIME + Kokkos::fence(); +#endif + copy_last_location(particle_destinations, size); + copy_material_ids(material_ids, size / 3); +} + +void PumiTallyImpl::copy_last_location(double *particle_destination, + int64_t size) { + auto last_location = p_pumi_particle_at_elem_boundary_handler->prev_xpoint_; + OMEGA_H_CHECK_PRINTF(last_location.size() >= size, + "last_location size %d is not greater to %ld\n", + last_location.size(), size); + Kokkos::View> + device_last_location_view(last_location.data(), size); + Kokkos::View> + host_last_location_view(particle_destination, size); + + Kokkos::deep_copy(host_last_location_view, device_last_location_view); +} + +void PumiTallyImpl::copy_material_ids(int *material_ids, int64_t size) { + auto material_ids_l = p_pumi_particle_at_elem_boundary_handler->material_ids_; + OMEGA_H_CHECK_PRINTF(material_ids_l.size() >= size, + "material_ids_l size %d is not greater to %ld\n", + material_ids_l.size(), size); + Kokkos::View> + host_material_ids_view(material_ids, size); + Kokkos::View> + device_material_ids_view(material_ids_l.data(), size); + + Kokkos::deep_copy(host_material_ids_view, device_material_ids_view); +} + +void PumiTallyImpl::write_pumi_tally_mesh() { + p_pumi_particle_at_elem_boundary_handler->finalizeAndWritePumiFlux( + full_mesh_, "fluxresult.vtk"); +#ifdef PUMI_MEASURE_TIME + Kokkos::fence(); +#endif +} + +void PumiTallyImpl::copy_and_reset_flying_flag(int8_t *flying) { + // todo get the size too + auto device_in_adv_que_l = device_in_adv_que_; + Kokkos::View> + host_flying_view(flying, pumi_ps_size); + Kokkos::View> + device_flying_view(device_in_adv_que_l.data(), + device_in_adv_que_l.size()); + Kokkos::deep_copy(device_flying_view, host_flying_view); + + for (int64_t pid = 0; pid < pumi_ps_size; ++pid) { + // reset flying flag to zero + flying[pid] = 0; + } +} + +void PumiTallyImpl::copy_weights(double *weights) { + auto weights_l = weights_; + Kokkos::View> + host_weights_view(weights, pumi_ps_size); + Kokkos::View> + device_weights_view(weights_l.data(), weights_l.size()); + + Kokkos::deep_copy(device_weights_view, host_weights_view); + + auto p_wgt = pumipic_ptcls->get<4>(); + auto copy_particle_weights = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + p_wgt(pid) = weights_l[pid]; + }; + pumipic::parallel_for(pumipic_ptcls.get(), copy_particle_weights, + "copy particle weights"); +} + +void PumiTallyImpl::copy_groups(int *groups) { + auto groups_l = groups_; + Kokkos::View> + host_groups_view(groups, pumi_ps_size); + Kokkos::View> + device_groups_view(groups_l.data(), groups_l.size()); + + Kokkos::deep_copy(device_groups_view, host_groups_view); + auto p_groups = pumipic_ptcls->get<5>(); + auto copy_particle_groups = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + p_groups(pid) = groups_l[pid]; + }; + pumipic::parallel_for(pumipic_ptcls.get(), copy_particle_groups, + "copy particle groups"); +} + +void PumiTallyImpl::search_initial_elements() { // assign the location to ptcl + // dest + auto particle_dest = pumipic_ptcls->get<1>(); + auto in_flight = pumipic_ptcls->get<3>(); + + int64_t pumi_ps_size_l = pumi_ps_size; + const auto &device_pos_buffer_l = device_pos_buffer_; + + auto set_particle_dest = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if (mask > 0 && pid < pumi_ps_size_l) { + particle_dest(pid, 0) = device_pos_buffer_l[pid * 3 + 0]; + particle_dest(pid, 1) = device_pos_buffer_l[pid * 3 + 1]; + particle_dest(pid, 2) = device_pos_buffer_l[pid * 3 + 2]; + + // everyone is in flight for this initial search + in_flight(pid) = 1; + } + }; + pumipic::parallel_for(pumipic_ptcls.get(), set_particle_dest, + "set initial position as dest"); + + // *initial* build and search to find the initial elements of the particles + search_and_rebuild(true, true); + is_pumipic_initialized = true; +} + +void PumiTallyImpl::copy_data_to_device(double *init_particle_positions) { + // fixme it should get size too to avoid memory error + Kokkos::View> + host_pos_view(init_particle_positions, pumi_ps_size * 3); + + Kokkos::View> + device_pos_view(device_pos_buffer_.data(), pumi_ps_size * 3); + + Kokkos::deep_copy(device_pos_view, host_pos_view); +} + +// methods for PumiTallyImpl and PumiParticleAtElemBoundary + +void PumiTallyImpl::init_pumi_libs(int &argc, char **&argv) { + pp_lib = std::make_unique(&argc, &argv); + oh_lib = pp_lib->omega_h_lib(); +} + +[[deprecated("Use move_to_next_element which is appropriate for new search " + "class in pumipic. [!Note] It my show this even though it is not " + "used due to template instantiation.")]] +void pp_move_to_new_element(Omega_h::Mesh &mesh, PPPS *ptcls, + Omega_h::Write &elem_ids, + Omega_h::Write &ptcl_done, + Omega_h::Write &lastExit) { + const int dim = mesh.dim(); + const auto &face2elems = mesh.ask_up(dim - 1, dim); + const auto &face2elemElem = face2elems.ab2b; + const auto &face2elemOffset = face2elems.a2ab; + const auto in_flight = ptcls->get<3>(); + + auto set_next_element = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if (mask > 0 && !ptcl_done[pid] && in_flight(pid)) { + auto searchElm = elem_ids[pid]; + auto bridge = lastExit[pid]; + auto e2f_first = face2elemOffset[bridge]; + auto e2f_last = face2elemOffset[bridge + 1]; + auto upFaces = e2f_last - e2f_first; + assert(upFaces == 2); + auto faceA = face2elemElem[e2f_first]; + auto faceB = face2elemElem[e2f_first + 1]; + assert(faceA != faceB); + assert(faceA == searchElm || faceB == searchElm); + auto nextElm = (faceA == searchElm) ? faceB : faceA; + elem_ids[pid] = nextElm; + } + }; + parallel_for(ptcls, set_next_element, "pumipic_set_next_element"); +} + +void move_to_next_element(PPPS *ptcls, Omega_h::Write &elem_ids, + Omega_h::Write &next_elems) { + auto in_flight = ptcls->get<3>(); + auto move_to_next = PS_LAMBDA(const int e, const int pid, const int mask) { + // move only if particle in flight and not leaving the domain + if (mask > 0 && in_flight(pid) && next_elems[pid] != -1) { + elem_ids[pid] = next_elems[pid]; + } + }; + pumipic::parallel_for(ptcls, move_to_next, "move to next element"); +} + +void apply_boundary_condition(Omega_h::Mesh &mesh, PPPS *ptcls, + Omega_h::Write &elem_ids, + Omega_h::Write &next_elems, + Omega_h::Write &ptcl_done, + Omega_h::Write &lastExit, + Omega_h::Write &xFace, + Omega_h::Write &inter_points, + Omega_h::Write material_ids, bool initial) { + + // TODO: make this a member variable of the struct + auto particle_destination = ptcls->get<1>(); + const auto class_ids = mesh.get_array(3, "class_id"); + auto checkExposedEdges = + PS_LAMBDA(const int e, const int pid, const int mask) { + if (mask > 0 && !ptcl_done[pid]) { + bool reached_destination = (lastExit[pid] == -1); + bool hit_boundary = ((next_elems[pid] == -1) && (elem_ids[pid] != -1)); + + // for the initial run, we need to find the initial position of the + // particles + if (!initial) { // stop at geometry boundary + if (next_elems[pid] != -1) { + if (class_ids[elem_ids[pid]] != + class_ids[next_elems[pid]]) { // particle crosses geometry + // boundary + hit_boundary = true; + material_ids[pid] = class_ids[next_elems[pid]]; + } + } else { + material_ids[pid] = -1; // no material id if not in an element + } + } + + ptcl_done[pid] = + (reached_destination || hit_boundary) ? 1 : ptcl_done[pid]; + // assert that if the next element is -1, then the material id is -1 + if (!initial) { + if (next_elems[pid] == -1) { + OMEGA_H_CHECK_PRINTF( + material_ids[pid] == -1, + "Error: next_elems[%d] is -1 but material_ids[%d] " + "is %d\n", + pid, pid, material_ids[pid]); + } + // printf("Pid %d next element %d, elem id %d, material id %d\n", + // pid, next_elems[pid], elem_ids[pid], material_ids[pid]); + } + + if (hit_boundary) { // just reached the boundary + // printf("Moving particle %4d (from -> to): element (%5d -> %5d) + // Material (%3d -> %3d)\n", + // pid, elem_ids[pid], next_elems[pid], + // class_ids[elem_ids[pid]], material_ids[pid]); + xFace[pid] = lastExit[pid]; + // particle reaches the boundary + particle_destination(pid, 0) = inter_points[pid * 3]; + particle_destination(pid, 1) = inter_points[pid * 3 + 1]; + particle_destination(pid, 2) = inter_points[pid * 3 + 2]; + } + } + }; + pumipic::parallel_for(ptcls, checkExposedEdges, + "apply vacumm boundary condition"); +} + +void PumiParticleAtElemBoundary::initialize_flux_array(size_t nelems, + size_t nEgroups) { + Kokkos::resize(flux_, nelems, nEgroups, 3); + auto flux_l = flux_; + OMEGA_H_CHECK(flux_l.extent(0) == nelems); + OMEGA_H_CHECK(flux_l.extent(1) == nEgroups); + OMEGA_H_CHECK(flux_l.extent(2) == 3); +} + +PumiParticleAtElemBoundary::PumiParticleAtElemBoundary(size_t nelems, + size_t capacity) + : prev_xpoint_(capacity * 3, 0.0, "prev_xpoint"), + material_ids_(capacity, -1, "material_ids"), + total_tracklength_(capacity, 0.0, "total_tracklength"), initial_(true) { + initialize_flux_array(nelems, 2); // FIXME: hardcoded 2 groups + printf("[INFO] Particle handler at boundary with %d elements and %d " + "x points size (3 * n_particles)\n", + flux_.size(), prev_xpoint_.size()); +} + +void PumiParticleAtElemBoundary::operator()( + Omega_h::Mesh &mesh, pumiinopenmc::PPPS *ptcls, + Omega_h::Write &elem_ids, + Omega_h::Write &next_elems, + Omega_h::Write &inter_faces, + Omega_h::Write &lastExit, + Omega_h::Write &inter_points, + Omega_h::Write &ptcl_done, + typeof(ptcls->get<0>()) origin_segment, + typeof(ptcls->get<1>()) dest_segment) { + if (!initial_) { + evaluateFlux(ptcls, inter_points, elem_ids, ptcl_done); + updatePrevXPoint(inter_points); + } + apply_boundary_condition(mesh, ptcls, elem_ids, next_elems, ptcl_done, + lastExit, inter_faces, inter_points, material_ids_, + initial_); + move_to_next_element(ptcls, elem_ids, next_elems); +} + +void PumiParticleAtElemBoundary::mark_initial_as(bool initial) { + initial_ = initial; +} + +void PumiParticleAtElemBoundary::updatePrevXPoint( + Omega_h::Write &xpoints) { + OMEGA_H_CHECK_PRINTF(xpoints.size() <= prev_xpoint_.size() && + prev_xpoint_.size() != 0, + "xpoints size %d is greater than prev_xpoint size %d\n", + xpoints.size(), prev_xpoint_.size()); + auto &prev_xpoint = prev_xpoint_; + auto update = OMEGA_H_LAMBDA(Omega_h::LO i) { prev_xpoint[i] = xpoints[i]; }; + Omega_h::parallel_for(xpoints.size(), update, "update previous xpoints"); +} + +void PumiParticleAtElemBoundary::updatePrevXPoint(PPPS *ptcls) { + // todo add checks of size + auto prev_xpoints_l = prev_xpoint_; + OMEGA_H_CHECK_PRINTF( + ptcls->capacity() * 3 == prev_xpoints_l.size(), + "Error: prev_xpoints_s are not size properly capacity %d size %d\n", + ptcls->capacity(), prev_xpoints_l.size()); + auto xpoints = ptcls->get<0>(); + auto update = PS_LAMBDA(const auto &e, const auto &pid, const auto &mask) { + prev_xpoints_l[pid * 3 + 0] = xpoints(pid, 0); + prev_xpoints_l[pid * 3 + 1] = xpoints(pid, 1); + prev_xpoints_l[pid * 3 + 2] = xpoints(pid, 2); + }; + pumipic::parallel_for(ptcls, update, + "update previous xpoints from origin points"); +} + +void PumiParticleAtElemBoundary::evaluateFlux( + PPPS *ptcls, Omega_h::Write xpoints, + Omega_h::Write elem_ids, + Omega_h::Write ptcl_done) { + // Omega_h::Real total_particles = ptcls->nPtcls(); + auto prev_xpoint = prev_xpoint_; + auto flux = flux_; + auto total_tracklength_l = total_tracklength_; + auto in_flight = ptcls->get<3>(); + auto p_wgt = ptcls->get<4>(); + auto p_groups = ptcls->get<5>(); + auto xpoints_l = xpoints; // todo shouldn't need it, so remove + + auto evaluate_flux = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if ((mask > 0) && (in_flight(pid) == 1) && !ptcl_done[pid]) { + OMEGA_H_CHECK_PRINTF( + total_tracklength_l[pid] >= 0.0, + "ERROR: Particle is moving but the tracklength is negative: %.16f\n", + total_tracklength_l[pid]); + + Omega_h::Vector<3> dest = {xpoints_l[pid * 3 + 0], xpoints_l[pid * 3 + 1], + xpoints_l[pid * 3 + 2]}; + Omega_h::Vector<3> orig = {prev_xpoint[pid * 3 + 0], + prev_xpoint[pid * 3 + 1], + prev_xpoint[pid * 3 + 2]}; + + Omega_h::Real segment_length = + Omega_h::norm(dest - orig); // / total_particles; + if (segment_length > + total_tracklength_l[pid] + 1e-6) { // tol for float operations and + // search algorithm's inaccuracy + // fixme: something wrong with orig: previous_xpoints are incorrect at + // least for the first run + printf("ERROR: Segment length in an element cannot be greater than the " + "total tracklength but found %.16f, %.16f of pid %d crossing el " + "%d starting at %d\nOrig: (%.16f, %.16f, %.16f), Dest: (%.16f, " + "%.16f, %.16f)\n", + segment_length, total_tracklength_l[pid], pid, elem_ids[pid], e, + orig[0], orig[1], orig[2], dest[0], dest[1], dest[2]); + } + + Omega_h::Real contribution = segment_length * p_wgt(pid); + + int group = p_groups(pid); + OMEGA_H_CHECK_PRINTF( + group >= 0 && group < flux.extent(1), + "ERROR: Group %d is out of bounds for flux array with " + "extent %lu\n", + group, flux.extent(1)); + + Kokkos::atomic_add(&flux(elem_ids[pid], group, 0), contribution); + Kokkos::atomic_add(&flux(elem_ids[pid], group, 1), + contribution * contribution); + } + }; + pumipic::parallel_for(ptcls, evaluate_flux, "flux evaluation loop"); +} + +void PumiParticleAtElemBoundary::normalizeFlux(Omega_h::Mesh &mesh) { + const Omega_h::LO nelems = mesh.nelems(); + const auto &el2n = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; + const auto &coords = mesh.coords(); + + auto flux = flux_; + auto total_tracklength_l = total_tracklength_; + + Omega_h::Write tet_volumes(flux_.extent(0), -1.0, + "tet_volumes"); + Omega_h::Write sd(flux_.extent(0), -1.0, "sd"); + + auto normalize_flux_with_volume = OMEGA_H_LAMBDA(Omega_h::LO elem_id) { + const auto elem_verts = Omega_h::gather_verts<4>(el2n, elem_id); + const auto elem_vert_coords = + Omega_h::gather_vectors<4, 3>(coords, elem_verts); + + auto b = Omega_h::simplex_basis<3, 3>(elem_vert_coords); + auto volume = Omega_h::simplex_size_from_basis(b); + + tet_volumes[elem_id] = volume; + for (int g = 0; g < flux.extent(1); g++) { + flux(elem_id, g, 0) /= (volume * total_tracklength_l.size()); + flux(elem_id, g, 1) /= (volume * volume * total_tracklength_l.size()); + + // calculate standard deviation FIXME this is not correct, needs number of + // iterations + flux(elem_id, g, 2) = Kokkos::sqrt( + flux(elem_id, g, 1) - flux(elem_id, g, 0) * flux(elem_id, g, 0)); + } + }; + Omega_h::parallel_for(tet_volumes.size(), normalize_flux_with_volume, + "normalize flux"); + + mesh.add_tag(Omega_h::REGION, "volume", 1, Omega_h::Reals(tet_volumes)); +} + +void PumiParticleAtElemBoundary::finalizeAndWritePumiFlux( + Omega_h::Mesh &full_mesh, const std::string &filename) { + normalizeFlux(full_mesh); + auto flux = flux_; + int num_groups = flux.extent(1); + int num_elems = flux.extent(0); + Omega_h::Write normalized_flux(num_elems, 0.0, + "normalized_flux"); + // copy flux to normalized_flux + for (int g = 0; g < num_groups; g++) { + auto copy_flux = OMEGA_H_LAMBDA(Omega_h::LO elem_id) { + normalized_flux[elem_id] = flux(elem_id, g, 0); + }; + Omega_h::parallel_for(num_elems, copy_flux, "copy flux to normalized_flux"); + std::string tag_name = "flux_group_" + std::to_string(g); + Kokkos::fence(); + full_mesh.add_tag(Omega_h::REGION, tag_name, 1, + Omega_h::Reals(normalized_flux)); + } + Omega_h::vtk::write_parallel(filename, &full_mesh, 3); +} + +void pumiUpdatePtclPositions(PPPS *ptcls) { + auto x_ps_d = ptcls->get<0>(); + auto xtgt_ps_d = ptcls->get<1>(); + auto updatePtclPos = PS_LAMBDA(const int &, const int &pid, const bool &) { + x_ps_d(pid, 0) = xtgt_ps_d(pid, 0); + x_ps_d(pid, 1) = xtgt_ps_d(pid, 1); + x_ps_d(pid, 2) = xtgt_ps_d(pid, 2); + xtgt_ps_d(pid, 0) = 0.0; + xtgt_ps_d(pid, 1) = 0.0; + xtgt_ps_d(pid, 2) = 0.0; + }; + ps::parallel_for(ptcls, updatePtclPos); +} + +void PumiParticleAtElemBoundary::compute_total_tracklength(PPPS *ptcls) { + auto orig = ptcls->get<0>(); + auto dest = ptcls->get<1>(); + + auto total_tracklength_l = total_tracklength_; + + auto computeTrackLength = + PS_LAMBDA(const int &elemId, const int &pid, const bool &mask) { + Omega_h::Vector<3> p_orig = {orig(pid, 0), orig(pid, 1), orig(pid, 2)}; + Omega_h::Vector<3> p_dest = {dest(pid, 0), dest(pid, 1), dest(pid, 2)}; + Omega_h::Real track_length = Omega_h::norm(p_dest - p_orig); + total_tracklength_l[pid] = track_length; + }; + pumipic::parallel_for(ptcls, computeTrackLength, + "compute total track length"); +} + +// search and update parent elements +//! @param initial initial search finds the initial location of the particles +//! and doesn't tally +void PumiTallyImpl::search_and_rebuild(bool initial, const bool migrate) { + // initial cannot be false when is_pumipic_initialized is false + // may fail if simulated more than one batch + assert((is_pumipic_initialized == false && initial == true) || + (is_pumipic_initialized == true && initial == false)); + p_pumi_particle_at_elem_boundary_handler->mark_initial_as(initial); + auto orig = pumipic_ptcls->get<0>(); + auto dest = pumipic_ptcls->get<1>(); + auto pid = pumipic_ptcls->get<2>(); + + if (p_picparts_->mesh() == nullptr || p_picparts_->mesh()->nelems() == 0) { + fprintf(stderr, "ERROR: Mesh is empty\n"); + } + + // total tracklengths are used to calculate the flux + if (!initial) { + p_pumi_particle_at_elem_boundary_handler->updatePrevXPoint( + pumipic_ptcls.get()); + p_pumi_particle_at_elem_boundary_handler->compute_total_tracklength( + pumipic_ptcls.get()); + } + + bool isFoundAll = p_particle_tracer_->search(migrate); + + if (!isFoundAll) { + printf( + "ERROR: Not all particles are found. May need more loops in search\n"); + } +} + +std::unique_ptr pp_create_particle_structure(Omega_h::Mesh mesh, + pumipic::lid_t numPtcls) { + Omega_h::Int ne = mesh.nelems(); + pumiinopenmc::PPPS::kkLidView ptcls_per_elem("ptcls_per_elem", ne); + pumiinopenmc::PPPS::kkGidView element_gids("element_gids", ne); + + Kokkos::TeamPolicy policy; + + Omega_h::parallel_for( + ne, OMEGA_H_LAMBDA(const Omega_h::LO &i) { element_gids(i) = i; }); + + Omega_h::parallel_for( + mesh.nelems(), OMEGA_H_LAMBDA(Omega_h::LO id) { + ptcls_per_elem[id] = (id == 0) ? numPtcls : 0; + }); + +#ifdef PUMI_USE_KOKKOS_CUDA + printf("PumiPIC Using GPU for simulation...\n"); + policy = Kokkos::TeamPolicy(10000, 32); +#else + printf("PumiPIC Using CPU for simulation...\n"); + policy = + Kokkos::TeamPolicy(10000, Kokkos::AUTO()); +#endif + + auto ptcls = std::make_unique>( + policy, ne, numPtcls, ptcls_per_elem, element_gids); + + return ptcls; +} + +/* +void set_pumipic_particle_structure_size(int openmc_particles_in_flight, int +openmc_work_per_rank, int openmc_n_particles) +{ + int64_t n_particles; // TODO have a better way to do it than this + // FIXME why this work_per rank is not set in the settings by now? + + + if (openmc_particles_in_flight == 0 && openmc_work_per_rank == 0) { + printf("While creating PumiPIC particle structure, both +max_particles_in_flight and work_per_rank are 0.\n"); n_particles = +(openmc_n_particles != 0) ? openmc_n_particles : pumi_ps_size; }else if +(openmc_particles_in_flight == 0 || openmc_work_per_rank == 0) { n_particles = +std::max(openmc_particles_in_flight, openmc_work_per_rank); printf("One of +max_particles_in_flight or work_per_rank is 0. Setting PumiPIC particle +structure size to %d\n", n_particles); } else { n_particles = +std::min(openmc_particles_in_flight, openmc_work_per_rank); printf("Setting +PumiPIC particle structure size to %d\n", n_particles); + } + + pumi_ps_size = n_particles; + printf("Creteating PumiPIC particle structure with size %d\n", n_particles); +} +*/ + +void start_pumi_particles_in_0th_element(Omega_h::Mesh &mesh, + pumiinopenmc::PPPS *ptcls) { + // find the centroid of the 0th element + const auto &coords = mesh.coords(); + const auto &tet2node = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; + + Omega_h::Write centroid_of_el0(3, 0.0, "centroid"); + + auto find_centroid_of_el0 = OMEGA_H_LAMBDA(Omega_h::LO id) { + const auto nodes = Omega_h::gather_verts<4>(tet2node, id); + Omega_h::Few, 4> tet_node_coords = + Omega_h::gather_vectors<4, 3>(coords, nodes); + const auto centroid = o::average(tet_node_coords); + centroid_of_el0[0] = centroid[0]; + centroid_of_el0[1] = centroid[1]; + centroid_of_el0[2] = centroid[2]; + }; + Omega_h::parallel_for(1, find_centroid_of_el0, "find centroid of element 0"); + + // assign the location to all particles + auto init_loc = ptcls->get<0>(); + auto pids = ptcls->get<2>(); + auto in_fly = ptcls->get<3>(); + + auto set_initial_positions = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if (mask > 0) { + pids(pid) = pid; + in_fly(pid) = 1; + init_loc(pid, 0) = centroid_of_el0[0]; + init_loc(pid, 1) = centroid_of_el0[1]; + init_loc(pid, 2) = centroid_of_el0[2]; + } + }; + pumipic::parallel_for(ptcls, set_initial_positions, + "set initial particle positions"); +} + +Omega_h::Mesh *PumiTallyImpl::partition_pumipic_mesh() { + Omega_h::Write owners(full_mesh_.nelems(), 0, "owners"); + // all the particles are initialized in element 0 to do an initial search to + // find the starting locations + // of the openmc given particles. + // p_picparts_ = new pumipic::Mesh(full_mesh_, Omega_h::LOs(owners)); + p_picparts_ = + std::make_unique(full_mesh_, Omega_h::LOs(owners)); + printf("PumiPIC mesh partitioned\n"); + Omega_h::Mesh *mesh = p_picparts_->mesh(); + return mesh; +} + +void PumiTallyImpl::create_and_initialize_pumi_particle_structure( + Omega_h::Mesh *mesh) { + pumipic_ptcls = pp_create_particle_structure(*mesh, pumi_ps_size); + start_pumi_particles_in_0th_element(*mesh, pumipic_ptcls.get()); + p_pumi_particle_at_elem_boundary_handler = + std::make_unique( + mesh->nelems(), pumipic_ptcls->capacity()); + + printf("PumiPIC Mesh and data structure created with %d and %d as particle " + "structure capacity\n", + p_picparts_->mesh()->nelems(), pumipic_ptcls->capacity()); +} + +void PumiTallyImpl::read_pumipic_lib_and_full_mesh(int &argc, char **&argv) { + printf("Reading the Omega_h mesh %s to tally with tracklength estimator\n", + oh_mesh_fname.c_str()); + init_pumi_libs(argc, argv); + + if (oh_mesh_fname.empty()) { + printf("[ERROR] Omega_h mesh for PumiPIC is not given. Provide --ohMesh = " + ""); + } + full_mesh_ = Omega_h::binary::read(oh_mesh_fname, &oh_lib); + if (full_mesh_.dim() != 3) { + printf("PumiPIC only works for 3D mesh now.\n"); + } + OMEGA_H_CHECK_PRINTF(full_mesh_.has_tag(Omega_h::REGION, "class_id"), + "Mesh %s does not have class_id tag on regions.\n", + oh_mesh_fname.c_str()); + printf("PumiPIC Loaded mesh %s with %d elements\n", oh_mesh_fname.c_str(), + full_mesh_.nelems()); +} + +void PumiTallyImpl::load_pumipic_mesh_and_init_particles(int &argc, + char **&argv) { + read_pumipic_lib_and_full_mesh(argc, argv); + Omega_h::Mesh *mesh = partition_pumipic_mesh(); + create_and_initialize_pumi_particle_structure(mesh); +} + +} // namespace pumiinopenmc diff --git a/src/pumitallyopenmc/pumitally_impl.hpp b/src/pumitallyopenmc/pumitally_impl.hpp new file mode 100644 index 0000000..993d6dd --- /dev/null +++ b/src/pumitallyopenmc/pumitally_impl.hpp @@ -0,0 +1,170 @@ +#ifndef PUMITALLYOPENMC_PUMITALLY_IMPL_HPP +#define PUMITALLYOPENMC_PUMITALLY_IMPL_HPP + +// TODO: rename the namespace and even the project to pumitally +// openmc is not necessary in the name since it is general purpose + +#include +#include +#include +#include +#include +#include + +namespace pumiinopenmc { +struct TallyTimes { + double initialization_time = 0.0; + double total_time_to_tally = 0.0; + double vtk_file_write_time = 0.0; + + void print_times() const; +}; + +// ------------------------------------------------------------------------------------------------// +// * Data structure for PumiPic +// Particle: 0-origin, 1-destination, 2-particle_id, +// 3-in_advance_particle_queue, 4-weight, 5-group +typedef pumipic::MemberTypes + PPParticle; +typedef pumipic::ParticleStructure PPPS; +typedef Kokkos::DefaultExecutionSpace PPExeSpace; + +// ------------------------------------------------------------------------------------------------// +// * Helper Functions +[[deprecated("Use move_to_next_element which is appropriate for new search " + "class in pumipic. [!Note] It my show this even though it is not " + "used due to template instantiation.")]] +void pp_move_to_new_element(Omega_h::Mesh &mesh, PPPS *ptcls, + Omega_h::Write &elem_ids, + Omega_h::Write &ptcl_done, + Omega_h::Write &lastExit); + +std::unique_ptr pp_create_particle_structure(Omega_h::Mesh mesh, + pumipic::lid_t numPtcls); + +void start_pumi_particles_in_0th_element(Omega_h::Mesh &mesh, + pumiinopenmc::PPPS *ptcls); + +// ------------------------------------------------------------------------------------------------// +// * Struct for PumiParticleAtElemBoundary +class PumiParticleAtElemBoundary { +public: + PumiParticleAtElemBoundary(size_t nelems, size_t capacity); + + void operator()(Omega_h::Mesh &mesh, pumiinopenmc::PPPS *ptcls, + Omega_h::Write &elem_ids, + Omega_h::Write &next_elems, + Omega_h::Write &inter_faces, + Omega_h::Write &lastExit, + Omega_h::Write &inter_points, + Omega_h::Write &ptcl_done, + typeof(ptcls->get<0>()) origin_segment, + typeof(ptcls->get<1>()) dest_segment); + + void updatePrevXPoint(Omega_h::Write &xpoints); + + void updatePrevXPoint(PPPS *ptcls); + + void evaluateFlux(PPPS *ptcls, Omega_h::Write xpoints, + Omega_h::Write elem_ids, + Omega_h::Write ptcl_done); + + void finalizeAndWritePumiFlux(Omega_h::Mesh &full_mesh, + const std::string &filename); + + void normalizeFlux(Omega_h::Mesh &mesh); + + void mark_initial_as(bool initial); + + void compute_total_tracklength(PPPS *ptcls); + void initialize_flux_array(size_t nelems, size_t nEgroups); + + bool initial_; // in initial run, flux is not tallied + // Dims: nelems, nEgroups, 3(flux, flux^2, standard deviation) + Kokkos::View flux_; + Omega_h::Write prev_xpoint_; + Omega_h::Write total_tracklength_; + Omega_h::Write material_ids_; +}; +// ------------------------------------------------------------------------------------------------// + +// ------------------------------------------------------------------------------------------------// +// * Struct for PumiTallyImpl +class PumiTallyImpl { +public: + int64_t pumi_ps_size = 1000000; // hundred thousand + std::string oh_mesh_fname; + + Omega_h::Library oh_lib; + Omega_h::Mesh full_mesh_; + + std::unique_ptr pp_lib = nullptr; + std::unique_ptr p_picparts_ = nullptr; + std::unique_ptr pumipic_ptcls = nullptr; + + long double pumipic_tol = 1e-8; + bool is_pumipic_initialized = false; + int64_t iter_count_ = 0; + double total_initial_weight_ = 0.0; + + std::unique_ptr + p_pumi_particle_at_elem_boundary_handler; + std::unique_ptr< + ParticleTracer> + p_particle_tracer_; + + Omega_h::Write device_pos_buffer_; + Omega_h::Write device_in_adv_que_; + Omega_h::Write weights_; + Omega_h::Write groups_; + + TallyTimes tally_times; + + // * Constructor + PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, int &argc, + char **&argv); // fixme extra & + + // * Destructor + ~PumiTallyImpl() = default; + + // Functions + void create_and_initialize_pumi_particle_structure(Omega_h::Mesh *mesh); + + void load_pumipic_mesh_and_init_particles(int &argc, char **&argv); + + Omega_h::Mesh *partition_pumipic_mesh(); + + void init_pumi_libs(int &argc, char **&argv); + + void search_and_rebuild(bool initial, bool migrate = true); + + void read_pumipic_lib_and_full_mesh(int &argc, char **&argv); + + void initialize_particle_location(double *init_particle_positions, + int64_t size); + + void move_to_next_location(double *particle_destinations, int8_t *flying, + double *weights, int *groups, int *material_ids, + int64_t size); + + void write_pumi_tally_mesh(); + + void copy_data_to_device(double *init_particle_positions); + + void search_initial_elements(); + + void copy_and_reset_flying_flag(int8_t *flying); + + void copy_weights(double *weights); + + void copy_groups(int *groups); + + void copy_last_location(double *particle_destination, int64_t size); + + void copy_material_ids(int *material_ids, int64_t size); +}; + +} // namespace pumiinopenmc + +#endif // PUMITALLYOPENMC_PUMITALLY_IMPL_HPP diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3d3ef1b..5d87ec8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -26,7 +26,7 @@ catch_discover_tests(test_hostwrite_initialization) add_executable(test_pumi_tally_impl_methods test_pumi_tally_impl_methods.cpp) target_include_directories(test_pumi_tally_impl_methods PRIVATE ${PROJECT_SOURCE_DIR}/src/pumitallyopenmc) -target_link_libraries(test_pumi_tally_impl_methods PUBLIC Catch2::Catch2WithMain pumipic::pumipic) +target_link_libraries(test_pumi_tally_impl_methods PUBLIC Catch2::Catch2WithMain pumitallyopenmc pumipic::pumipic) catch_discover_tests(test_pumi_tally_impl_methods) if(PUMI_ENABLE_DEGAS2) diff --git a/test/test_pumi_tally_impl_methods.cpp b/test/test_pumi_tally_impl_methods.cpp index 89057c1..4d68fe3 100644 --- a/test/test_pumi_tally_impl_methods.cpp +++ b/test/test_pumi_tally_impl_methods.cpp @@ -1,6 +1,7 @@ // // Created by Fuad Hasan on 2/3/25. // +#include "pumitally_impl.hpp" #include #include #include @@ -11,14 +12,10 @@ // - Sections are not working for some reason. Saying using MPI functions before // or after MPI init or finalize // - sections are marked with comments for now -// - TODO Remove including cpp by creating another internal header // - Look at this gist to verify this in python: // https://gist.github.com/Fuad-HH/5e0aed99f271617e283e9108091fb1cb // *****************************************************************************************************// -// TODO: Remove it by having another header file -#include "pumipic_particle_data_structure.cpp" - bool is_close(const double a, const double b, double tol = 1e-8) { return std::abs(a - b) < tol; } From 59ed1f204ce07d3180e0821369c1fdb239e8c48a Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Mon, 14 Jul 2025 23:32:01 -0400 Subject: [PATCH 096/139] fix linking error - pumitallyopenmc library is linked with pumipic privately. Therefore, every other library or binary that uses pumipic, needs to be linked again. --- CMakeLists.txt | 5 +++-- src/physics/degas2/degas2-main.cpp | 6 ++---- src/pumitallyopenmc/pumipic_particle_data_structure.cpp | 8 +++++++- src/pumitallyopenmc/pumitally_impl.cpp | 7 +------ .../{pumitally_impl.hpp => pumitally_impl.tpp} | 6 +++--- test/test_pumi_tally_impl_methods.cpp | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) rename src/pumitallyopenmc/{pumitally_impl.hpp => pumitally_impl.tpp} (97%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 050a5d5..f1ce6f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ endif() ####################### Headers and Sources ####################### set(PUMITALLYOPENMC_HEADERS src/pumitallyopenmc/pumipic_particle_data_structure.h - src/pumitallyopenmc/pumitally_impl.hpp + src/pumitallyopenmc/pumitally_impl.tpp ) set(PUMITALLYOPENMC_SOURCES @@ -71,6 +71,7 @@ set(CMAKE_BUILD_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") ################################## Create Library ################# +# TODO: Create a different library for impl add_library(pumitallyopenmc) target_sources(pumitallyopenmc PRIVATE ${PUMITALLYOPENMC_SOURCES}) target_link_libraries(pumitallyopenmc PRIVATE pumipic::pumipic) @@ -96,7 +97,7 @@ if(PUMI_ENABLE_DEGAS2) find_package(HDF5 REQUIRED COMPONENTS CXX) add_library(pumitally_degas2) target_sources(pumitally_degas2 PRIVATE ${PUMI_DEGAS2_SOURCES}) - target_link_libraries(pumitally_degas2 PUBLIC Omega_h::omega_h HDF5::HDF5) + target_link_libraries(pumitally_degas2 PUBLIC pumipic::pumipic HDF5::HDF5) target_include_directories(pumitally_degas2 PUBLIC ${HDF5_INCLUDE_DIRS}) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index c67dff5..ffe52d4 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -5,7 +5,7 @@ */ #include "DG2Physics.h" -#include +#include "pumitally_impl.tpp" // ******************************************* Helper Functions // ******************************************* // @@ -21,9 +21,7 @@ int main(int argc, char *argv[]) { read_input_parameters(argc, argv, mesh_name, num_particles); print_initial_info(mesh_name, num_particles); - // Initialize PUMI-Tally - auto pumitally = - pumiinopenmc::PumiTally(mesh_name, num_particles, argc, argv); + pumiinopenmc::TallyTimes tally_times; return 0; } diff --git a/src/pumitallyopenmc/pumipic_particle_data_structure.cpp b/src/pumitallyopenmc/pumipic_particle_data_structure.cpp index 8b432b1..5e3243d 100644 --- a/src/pumitallyopenmc/pumipic_particle_data_structure.cpp +++ b/src/pumitallyopenmc/pumipic_particle_data_structure.cpp @@ -1,5 +1,5 @@ #include "pumipic_particle_data_structure.h" -#include "pumitally_impl.hpp" +#include "pumitally_impl.tpp" namespace pumiinopenmc { PumiTally::PumiTally(std::string &mesh_filename, int64_t num_particles, @@ -42,4 +42,10 @@ void PumiTally::write_pumi_tally_mesh() { pimpl->tally_times.vtk_file_write_time += elapsed_seconds.count(); pimpl->tally_times.print_times(); } + +PumiTally::~PumiTally() { + pimpl.reset(nullptr); + Kokkos::finalize(); +} + } // namespace pumiinopenmc diff --git a/src/pumitallyopenmc/pumitally_impl.cpp b/src/pumitallyopenmc/pumitally_impl.cpp index 97ec319..35c0997 100644 --- a/src/pumitallyopenmc/pumitally_impl.cpp +++ b/src/pumitallyopenmc/pumitally_impl.cpp @@ -2,8 +2,7 @@ // Created by Fuad Hasan on 12/3/24. // -#include "pumitally_impl.hpp" -#include "pumipic_particle_data_structure.h" +#include "pumitally_impl.tpp" #include #include #include @@ -19,10 +18,6 @@ void TallyTimes::print_times() const { printf("[TIME] Total PumiPic time : %f seconds\n", initialization_time + total_time_to_tally + vtk_file_write_time); } -PumiTally::~PumiTally() { - pimpl.reset(nullptr); - Kokkos::finalize(); -} PumiTallyImpl::PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, int &argc, char **&argv) { diff --git a/src/pumitallyopenmc/pumitally_impl.hpp b/src/pumitallyopenmc/pumitally_impl.tpp similarity index 97% rename from src/pumitallyopenmc/pumitally_impl.hpp rename to src/pumitallyopenmc/pumitally_impl.tpp index 993d6dd..fc693c7 100644 --- a/src/pumitallyopenmc/pumitally_impl.hpp +++ b/src/pumitallyopenmc/pumitally_impl.tpp @@ -1,5 +1,5 @@ -#ifndef PUMITALLYOPENMC_PUMITALLY_IMPL_HPP -#define PUMITALLYOPENMC_PUMITALLY_IMPL_HPP +#ifndef PUMITALLYOPENMC_PUMITALLY_IMPL_TPP +#define PUMITALLYOPENMC_PUMITALLY_IMPL_TPP // TODO: rename the namespace and even the project to pumitally // openmc is not necessary in the name since it is general purpose @@ -167,4 +167,4 @@ class PumiTallyImpl { } // namespace pumiinopenmc -#endif // PUMITALLYOPENMC_PUMITALLY_IMPL_HPP +#endif // PUMITALLYOPENMC_PUMITALLY_IMPL_TPP diff --git a/test/test_pumi_tally_impl_methods.cpp b/test/test_pumi_tally_impl_methods.cpp index 4d68fe3..2f3bec4 100644 --- a/test/test_pumi_tally_impl_methods.cpp +++ b/test/test_pumi_tally_impl_methods.cpp @@ -1,7 +1,7 @@ // // Created by Fuad Hasan on 2/3/25. // -#include "pumitally_impl.hpp" +#include "pumitally_impl.tpp" #include #include #include From bf1e8fc30ad09c2c0fd9f1992b03fb09c750d96f Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Tue, 15 Jul 2025 05:23:02 -0400 Subject: [PATCH 097/139] first draft of degas2 - contains all major components - Field read function template - **Not tested** --- src/physics/degas2/degas2-main.cpp | 338 +++++++++++++++++++++++++++-- 1 file changed, 323 insertions(+), 15 deletions(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index ffe52d4..93e3707 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -4,24 +4,116 @@ * PUMI-Tally interface */ +#include +#include +#include + #include "DG2Physics.h" #include "pumitally_impl.tpp" // ******************************************* Helper Functions // ******************************************* // -void read_input_parameters(int argc, char *const *argv, std::string &mesh_name, - int &num_particles); +enum class SourceDistribution { + UNIFORM, // Source uniformly distributed across the mesh + EQUAL // Source at centroids of each element +}; +struct InputParameters { + std::string mesh_name; + int num_particles = 0; + int max_iterations = 1000; + SourceDistribution source_distribution = SourceDistribution::EQUAL; +}; +void read_input_parameters(int argc, char *const *argv, + InputParameters ¶ms); void print_initial_info(const std::string &mesh_name, int num_particles) noexcept; +// computes centroids of each element in the mesh and stores them in a tag +void get_centroids(Omega_h::Mesh &mesh, + Omega_h::Write centroids); + +template +void vector2write(const std::vector &vec, Omega_h::Write &write_vec) { + Omega_h::HostWrite host_write_vec(vec.size()); + for (size_t i = 0; i < vec.size(); ++i) { + host_write_vec[i] = vec[i]; + } + write_vec = Omega_h::Write(host_write_vec); +} + +struct Fields { + std::vector electron_temperature; + std::vector ion_temperature; + std::vector electron_density; + std::vector ion_density; + std::vector bulk_flow_velocity; +}; +void get_field_values(Omega_h::Reals centroids, Fields &fields); +void set_field_values_to_mesh(Omega_h::Mesh &mesh, const Fields &fields); +void set_source_particles(Omega_h::Mesh &mesh, + SourceDistribution source_distribution, + Omega_h::Write particle_positions); +void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, + const Omega_h::Read &electron_density, + const Omega_h::Read ion_density, + const Omega_h::Read electron_temperature, + const Omega_h::Read ion_temperature, + const Omega_h::Read bulk_flow_velocity, + int max_iterations); +// ***************************************************************************// +// *************************** Main Function ******************************** // +// ***************************************************************************// int main(int argc, char *argv[]) { - std::string mesh_name; - int num_particles; - read_input_parameters(argc, argv, mesh_name, num_particles); - print_initial_info(mesh_name, num_particles); + // Read input parameters + InputParameters input_params; + read_input_parameters(argc, argv, input_params); + print_initial_info(input_params.mesh_name, input_params.num_particles); + + // Initialize PUMI-Tally and Read Fields + auto pumi_tally = pumiinopenmc::PumiTallyImpl( + input_params.mesh_name, input_params.num_particles, argc, argv); + auto &mesh = pumi_tally.full_mesh_; + Omega_h::Write centroids(mesh.nelems() * 3); + get_centroids(mesh, centroids); + Fields fields; + get_field_values(Omega_h::Reals(centroids), fields); + set_field_values_to_mesh(mesh, fields); + + Kokkos::View sigma_t_; // mat, T, g + Kokkos::View sigma_a_; // mat, T, g + Kokkos::View scattering_matrix_; // mat, T, g, g + Kokkos::View sigma_s_; // mat, T, g + DG2CrossSection crossSection(sigma_t_, sigma_a_, scattering_matrix_, + sigma_s_); + DG2Physics physics(crossSection, input_params.num_particles); + + // Initialize particles + Omega_h::Write particle_positions(pumi_tally.pumi_ps_size * 3, + "particle_positions"); + set_source_particles(mesh, input_params.source_distribution, + particle_positions); + pumi_tally.device_pos_buffer_ = particle_positions; + pumi_tally.search_initial_elements(); + + // Move particles + auto electron_density = + mesh.get_array(Omega_h::REGION, "electron_density"); + auto ion_density = + mesh.get_array(Omega_h::REGION, "ion_density"); + auto electron_temperature = + mesh.get_array(Omega_h::REGION, "electron_temperature"); + auto ion_temperature = + mesh.get_array(Omega_h::REGION, "ion_temperature"); + auto bulk_flow_velocity = + mesh.get_array(Omega_h::REGION, "bulk_flow_velocity"); - pumiinopenmc::TallyTimes tally_times; + transport(pumi_tally, physics, electron_density, ion_density, + electron_temperature, ion_temperature, bulk_flow_velocity, + input_params.max_iterations); + + // Finalize and output results + pumi_tally.write_pumi_tally_mesh(); return 0; } @@ -33,6 +125,209 @@ int main(int argc, char *argv[]) { // **************************************************************************************************************** // // +void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, + const Omega_h::Read &electron_density, + const Omega_h::Read ion_density, + const Omega_h::Read electron_temperature, + const Omega_h::Read ion_temperature, + const Omega_h::Read bulk_flow_velocity, + int max_iterations) { + + for (int iter = 0; iter < max_iterations; ++iter) { + auto particle_dest = pumi_tally.pumipic_ptcls->get<1>(); + auto particle_orig = pumi_tally.pumipic_ptcls->get<0>(); + auto particle_weight = pumi_tally.pumipic_ptcls->get<4>(); + auto particle_group = pumi_tally.pumipic_ptcls->get<5>(); + auto get_new_destination = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if (mask > 0) { // FIXME: check if the particle is at destination or at + // the boundary + ParticleInfo particle_info; + particle_info.position[0] = particle_dest(pid, 0); + particle_info.position[1] = particle_dest(pid, 1); + particle_info.position[2] = particle_dest(pid, 2); + auto direction = Omega_h::normalize(Omega_h::Vector<3>{ + particle_info.position[0] - particle_orig(pid, 0), + particle_info.position[1] - particle_orig(pid, 1), + particle_info.position[2] - particle_orig(pid, 2)}); + particle_info.direction[0] = direction[0]; + particle_info.direction[1] = direction[1]; + particle_info.direction[2] = direction[2]; + particle_info.weight = particle_weight(pid); + particle_info.energy_group = particle_group(pid); + particle_info.particle_index = pid; + + FieldInfo field_info; + field_info.electron_temperature = electron_temperature[e]; + field_info.ion_temperature = ion_temperature[e]; + field_info.electron_density = electron_density[e]; + field_info.ion_density = ion_density[e]; + field_info.bulk_flow_velocity[0] = bulk_flow_velocity[e * 3 + 0]; + field_info.bulk_flow_velocity[1] = bulk_flow_velocity[e * 3 + 1]; + field_info.bulk_flow_velocity[2] = bulk_flow_velocity[e * 3 + 2]; + + physics.collide_particle(particle_info, field_info); + physics.sample_collision_distance(particle_info, field_info); + + // Update particle position and direction + particle_dest(pid, 0) = particle_info.position[0]; + particle_dest(pid, 1) = particle_info.position[1]; + particle_dest(pid, 2) = particle_info.position[2]; + particle_weight(pid) = particle_info.weight; + particle_group(pid) = particle_info.energy_group; + } + }; + pumipic::parallel_for(pumi_tally.pumipic_ptcls.get(), get_new_destination, + "get new destination"); + + pumi_tally.search_and_rebuild( + false, true); // for now, always rebuild the pp structure + Kokkos::fence(); + } +} + +void initialize_equal_source(const Omega_h::Mesh &mesh, + Omega_h::Write particle_positions) { + Omega_h::LO num_particles = particle_positions.size() / 3; + Omega_h::LO particles_per_element = num_particles / mesh.nelems(); + Omega_h::LO remainder_particles = num_particles % mesh.nelems(); + const auto centroids = + mesh.get_array(Omega_h::REGION, "centroid"); + + // set the particle positions to the centroids of each element, + // and adding one particle to the first few elements if there are remainder + // particles + Omega_h::parallel_for( + "set source particles", mesh.nelems(), OMEGA_H_LAMBDA(Omega_h::LO e) { + Omega_h::Vector<3> centroid = { + centroids[e * 3 + 0], centroids[e * 3 + 1], centroids[e * 3 + 2]}; + Omega_h::LO particles_in_this_element = + particles_per_element + (e < remainder_particles ? 1 : 0); + for (Omega_h::LO p = 0; p < particles_in_this_element; ++p) { + Omega_h::LO particle_index = + e * particles_per_element + p + + (e < remainder_particles ? e : remainder_particles); + particle_positions[particle_index * 3 + 0] = centroid[0]; + particle_positions[particle_index * 3 + 1] = centroid[1]; + particle_positions[particle_index * 3 + 2] = centroid[2]; + } + }); + Kokkos::fence(); +} + +void set_source_particles(Omega_h::Mesh &mesh, + const SourceDistribution source_distribution, + Omega_h::Write particle_positions) { + OMEGA_H_CHECK_PRINTF(particle_positions.size() > 0, + "Particle positions size (%ld) must be greater than 0\n", + particle_positions.size()); + + switch (source_distribution) { + case SourceDistribution::UNIFORM: + throw std::runtime_error("Uniform distribution is not implemented yet."); + break; + case SourceDistribution::EQUAL: { + initialize_equal_source(mesh, particle_positions); + + break; + } + default: + throw std::runtime_error("Unsupported source distribution!\n"); + } +} + +void set_field_values_to_mesh(Omega_h::Mesh &mesh, const Fields &fields) { + OMEGA_H_CHECK_PRINTF(fields.electron_temperature.size() == mesh.nelems(), + "Electron temperature size (%d) must be equal to number " + "of elements (%d)\n", + fields.electron_temperature.size(), mesh.nelems()); + OMEGA_H_CHECK_PRINTF( + fields.ion_temperature.size() == mesh.nelems(), + "Ion temperature size (%d) must be equal to number of elements (%d)\n", + fields.ion_temperature.size(), mesh.nelems()); + OMEGA_H_CHECK_PRINTF( + fields.electron_density.size() == mesh.nelems(), + "Electron density size (%d) must be equal to number of elements (%d)\n", + fields.electron_density.size(), mesh.nelems()); + OMEGA_H_CHECK_PRINTF( + fields.ion_density.size() == mesh.nelems(), + "Ion density size (%d) must be equal to number of elements (%d)\n", + fields.ion_density.size(), mesh.nelems()); + OMEGA_H_CHECK_PRINTF(fields.bulk_flow_velocity.size() == mesh.nelems() * 3, + "Bulk flow velocity size (%d) must be equal to number " + "of elements * 3 (%d)\n", + fields.bulk_flow_velocity.size(), mesh.nelems() * 3); + + Omega_h::Write electron_temperature( + fields.electron_temperature.size()); + Omega_h::Write ion_temperature(fields.ion_temperature.size()); + Omega_h::Write electron_density( + fields.electron_density.size()); + Omega_h::Write ion_density(fields.ion_density.size()); + Omega_h::Write bulk_flow_velocity( + fields.bulk_flow_velocity.size()); + vector2write(fields.electron_temperature, electron_temperature); + vector2write(fields.ion_temperature, ion_temperature); + vector2write(fields.electron_density, electron_density); + vector2write(fields.ion_density, ion_density); + vector2write(fields.bulk_flow_velocity, bulk_flow_velocity); + + // Assuming the mesh has a field for each of the properties + mesh.add_tag(Omega_h::REGION, "electron_temperature", 1, + electron_temperature); + mesh.add_tag(Omega_h::REGION, "ion_temperature", 1, + ion_temperature); + mesh.add_tag(Omega_h::REGION, "electron_density", 1, + electron_density); + mesh.add_tag(Omega_h::REGION, "ion_density", 1, ion_density); + mesh.add_tag(Omega_h::REGION, "bulk_flow_velocity", 3, + bulk_flow_velocity); +} + +void get_centroids(Omega_h::Mesh &mesh, + Omega_h::Write centroids) { + OMEGA_H_CHECK_PRINTF( + centroids.size() == mesh.nelems() * 3, + "Centroids size (%d) must be equal to number of elements * 3 (%d)\n", + centroids.size(), mesh.nelems() * 3); + const auto coords = mesh.coords(); + const auto nelems = mesh.nelems(); + const auto e2v = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; + + // FIXME: Hardcoded for 3D tets + Omega_h::parallel_for( + "calculate centroids", nelems, OMEGA_H_LAMBDA(int e) { + auto nodes = o::gather_verts<4>(e2v, e); + o::Few, 4> elem_coords = + o::gather_vectors<4, 3>(coords, nodes); + o::Vector<3> centroid = o::average(elem_coords); + + centroids[e * 3 + 0] = centroid[0]; + centroids[e * 3 + 1] = centroid[1]; + centroids[e * 3 + 2] = centroid[2]; + }); + + mesh.add_tag(Omega_h::REGION, "centroid", 3, centroids); +} + +// TODO: Implement the function to retrieve field values based on centroids +void get_field_values(Omega_h::Reals centroids, Fields &fields) { + auto centroids_h = Omega_h::HostRead(centroids); + + // This function should be implemented to retrieve the field values + // based on the centroids provided (read centroids_h on CPU. + // It contains centroids like e0_0, e0_1, e0_2, e1_0, e1_1, e1_2, ....) + // For now, we will just fill the fields with dummy data. + size_t num_elements = centroids.size() / 3; // Assuming 3D + fields.electron_temperature.resize(num_elements, 1.0); + fields.ion_temperature.resize(num_elements, 1.0); + fields.electron_density.resize(num_elements, 1.0); + fields.ion_density.resize(num_elements, 1.0); + fields.bulk_flow_velocity.resize(num_elements, 0.0); + + // TODO: Add your code here to retrieve the actual field values +} + void print_initial_info(const std::string &mesh_name, int num_particles) noexcept { printf("\n===================> Accelerated Degas2 <======================\n"); @@ -43,15 +338,28 @@ void print_initial_info(const std::string &mesh_name, printf("\n===============================================================\n"); } -void read_input_parameters(int argc, char *const *argv, std::string &mesh_name, - int &num_particles) { - if (argc != 3) { - throw std::runtime_error("Usage: " + std::string(argv[0]) + - " "); +void read_input_parameters(int argc, char *const *argv, + InputParameters ¶ms) { + if (argc != 5) { + throw std::runtime_error( + "Usage: " + std::string(argv[0]) + + " "); } - mesh_name = argv[1]; - num_particles = std::stoi(argv[2]); - if (num_particles <= 0) { + params.mesh_name = argv[1]; + params.num_particles = std::stoi(argv[2]); + if (params.num_particles <= 0) { throw std::runtime_error("Number of particles must be a positive integer."); } + params.max_iterations = std::stoi(argv[3]); + + std::string source_dist_str = argv[4]; + // FIXME: hardcoded lower case + if (source_dist_str == "uniform") { + params.source_distribution = SourceDistribution::UNIFORM; + } else if (source_dist_str == "equal") { + params.source_distribution = SourceDistribution::EQUAL; + } else { + throw std::runtime_error( + "Invalid source distribution. Use 'uniform' or 'equal'."); + } } From 4f9cd7e4a9275cba5560e6849c16c8327d8105f8 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 17 Jul 2025 13:27:28 -0400 Subject: [PATCH 098/139] Code in the "get_field_values" function --- src/physics/degas2/degas2-main.cpp | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 93e3707..031ece3 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -326,6 +326,38 @@ void get_field_values(Omega_h::Reals centroids, Fields &fields) { fields.bulk_flow_velocity.resize(num_elements, 0.0); // TODO: Add your code here to retrieve the actual field values + // Write centroid coords to CSV to be read by python script + std::ofstream coords("centroid_coords.csv"); + for (int j = 0; j < (centroids.size() / 3); j++) { + coords << j << "," << centroids[j*3 + 0] << "," << centroids[j*3 + 1] << "," << centroids[j*3 + 2] << std::endl; + } + coords.close(); + + // Run the python script + system("python Mesh_Map.py"); + + // Read in the field values + std::ifstream fieldvals{"vals.csv"}; + int i = 0; + std::string IDString; + std::string xxString; + std::string yyString; + std::string zzString; + + while (std::getline(fieldvals, IDString, ',')) { + + std::getline(fieldvals,xxString, ','); + std::getline(fieldvals,yyString, ','); + std::getline(fieldvals,zzString, '\n'); + + // Asign field values + fields.ion_density[i] = std::stod(xxString); + fields.electron_density[i] = std::stod(xxString); + fields.electron_temperature[i] = std::stod(yyString); + fields.ion_temperature[i] = std::stod(zzString); + + i++; + } } void print_initial_info(const std::string &mesh_name, From f168001369b93c8278216f123dfd9ce4a93ff411 Mon Sep 17 00:00:00 2001 From: Gabe Date: Thu, 17 Jul 2025 13:37:04 -0400 Subject: [PATCH 099/139] Add comments and add the headers for file I/O. --- src/physics/degas2/degas2-main.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 031ece3..2de7869 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "DG2Physics.h" #include "pumitally_impl.tpp" @@ -311,6 +313,9 @@ void get_centroids(Omega_h::Mesh &mesh, } // TODO: Implement the function to retrieve field values based on centroids +// NOTE: This requires the python script and the plasma source mesh and data +// to be in the folder with this cpp file. The python script may need to be +// updated to read in different plasma source data. void get_field_values(Omega_h::Reals centroids, Fields &fields) { auto centroids_h = Omega_h::HostRead(centroids); From f6ce9b6fc5fb8c13fcf6ec14efce144db224f1df Mon Sep 17 00:00:00 2001 From: gpaynter5004 Date: Fri, 18 Jul 2025 14:33:06 -0400 Subject: [PATCH 100/139] Update degas2-main.cpp Change centroids to centroids_h --- src/physics/degas2/degas2-main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 2de7869..419ca59 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -333,8 +333,8 @@ void get_field_values(Omega_h::Reals centroids, Fields &fields) { // TODO: Add your code here to retrieve the actual field values // Write centroid coords to CSV to be read by python script std::ofstream coords("centroid_coords.csv"); - for (int j = 0; j < (centroids.size() / 3); j++) { - coords << j << "," << centroids[j*3 + 0] << "," << centroids[j*3 + 1] << "," << centroids[j*3 + 2] << std::endl; + for (int j = 0; j < (centroids_h.size() / 3); j++) { + coords << j << "," << centroids_h[j*3 + 0] << "," << centroids_h[j*3 + 1] << "," << centroids_h[j*3 + 2] << std::endl; } coords.close(); From 2274d0bde21f61916121b3cce0a4f490db8b13e5 Mon Sep 17 00:00:00 2001 From: Gabe Date: Tue, 22 Jul 2025 15:22:15 -0400 Subject: [PATCH 101/139] Add alpha to particle_info struct to be multiplied into tally --- src/physics/degas2/DG2Physics.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index fb47cd8..148fa44 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -20,6 +20,7 @@ struct ParticleInfo { double weight; int energy_group; // Energy group *index* int particle_index; + double alpha; //To be multiplied in when computing tally }; struct FieldInfo { @@ -179,6 +180,8 @@ class DG2Physics { auto mag_v = Kokkos::sqrt(vx*vx + vy*vy + vz*vz); + particle_info.alpha = 1/mag_v; + particle_info.direction[0] = vx/mag_v; particle_info.direction[1] = vy/mag_v; particle_info.direction[2] = vz/mag_v; From 27c85cd92b1e964dd958d0a0ec1cbf405b24d8f8 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 23 Jul 2025 14:46:53 -0400 Subject: [PATCH 102/139] Implement rejection testing for the collision new velocity --- src/physics/degas2/DG2Physics.h | 51 ++++++++++++++++++++++++--------- test/test_degas2_physics.cpp | 5 ++-- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 148fa44..184ecda 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -134,6 +134,10 @@ class DG2Physics { return sigma_cx; } + KOKKOS_FUNCTION //IF THIS IS CHANGED MAY NEED TO CHANGE max_sigma_cx in the collision code + double analytic_cross_section_CE(double energy) const{ + return (0.6937e-14*(1-0.155*Kokkos::log10(energy))*(1-0.155*Kokkos::log10(energy)))/(1+0.1112e-14*Kokkos::pow(energy,3.3)); + } KOKKOS_FUNCTION void sample_collision_distance(ParticleInfo &particle_info, @@ -160,28 +164,47 @@ class DG2Physics { //Generate Random Numbers auto rand_gen = random_pool.get_state(); - double x1 = rand_gen.drand(0., 1.); - double x2 = rand_gen.drand(0., 1.); - double y1 = rand_gen.drand(0., 1.); - double y2 = rand_gen.drand(0., 1.); double ww = rand_gen.drand(0., 1.); - random_pool.free_state(rand_gen); double sigma_ion = ionization_cross_section(particle_info, field_info); double sigma_cx = charge_exchange_cross_section(particle_info, field_info); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) - double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - - auto vx = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::cos(2*M_PI*x2); - auto vy = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::sin(2*M_PI*x2); - auto vz = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(y1))*Kokkos::sin(2*M_PI*y2); - - auto mag_v = Kokkos::sqrt(vx*vx + vy*vy + vz*vz); + double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - particle_info.alpha = 1/mag_v; + bool rejection_test = false; + auto old_mag_v = Kokkos::sqrt(2*particle_energy(particle_info.particle_index)/mp); + double vx; + double vy; + double vz; + //Loops this until it passes the rejection test + while (!rejection_test) { + //Generate random numbers for the velocity + double x1 = rand_gen.drand(0., 1.); + double x2 = rand_gen.drand(0., 1.); + double y1 = rand_gen.drand(0., 1.); + double y2 = rand_gen.drand(0., 1.); + + //Sample a new velocity from the thermal distribution (cm/s) + vx = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::cos(2*M_PI*x2); + vy = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::sin(2*M_PI*x2); + vz = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(y1))*Kokkos::sin(2*M_PI*y2); + + //Compute the relative velocity + auto rel_vx = vx - particle_info.direction[0]*old_mag_v; + auto rel_vy = vy - particle_info.direction[1]*old_mag_v; + auto rel_vz = vz - particle_info.direction[2]*old_mag_v; + auto mag_v2 = rel_vx*rel_vx + rel_vy*rel_vy + rel_vz*rel_vz; + + //Generate random number and compare to sigma/sigma_max + if (rand_gen.drand(0., 1.) < analytic_cross_section_CE(0.5*mp*mag_v2)/max_sigma_cx) { + rejection_test = true; + } + } + //Now that the rejection test has passed, set the new information + double mag_v{Kokkos::sqrt(vx*vx+vy*vy+vz*vz)}; particle_info.direction[0] = vx/mag_v; particle_info.direction[1] = vy/mag_v; particle_info.direction[2] = vz/mag_v; @@ -206,6 +229,8 @@ class DG2Physics { } } particle_info.weight = new_weight; + random_pool.free_state(rand_gen); + } //To here diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 9279c88..28154e1 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -38,6 +38,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ particles(i).energy_group = energy; particles(i).weight = 1.0; particles(i).particle_index = i; + particles(i).alpha = 1.0; fields(i).electron_density = 1.0e13; fields(i).ion_density = 1.0e13; @@ -111,8 +112,8 @@ TEST_CASE("Test Degas2 Physics Functions"){ outfile << output(i).direction[2] << std::endl; } */ - REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.03)); - REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); + REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.04)); + REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.04)); REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); } From 70bf3e7d4ca0031ded70728769ff474b98b7c214 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 23 Jul 2025 14:51:02 -0400 Subject: [PATCH 103/139] Fix formatting and add comment --- src/physics/degas2/DG2Physics.h | 59 +++++++++++++++++---------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 184ecda..24addc9 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -174,37 +174,38 @@ class DG2Physics { double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 bool rejection_test = false; - auto old_mag_v = Kokkos::sqrt(2*particle_energy(particle_info.particle_index)/mp); - double vx; - double vy; - double vz; - //Loops this until it passes the rejection test - while (!rejection_test) { - //Generate random numbers for the velocity - double x1 = rand_gen.drand(0., 1.); - double x2 = rand_gen.drand(0., 1.); - double y1 = rand_gen.drand(0., 1.); - double y2 = rand_gen.drand(0., 1.); - - //Sample a new velocity from the thermal distribution (cm/s) - vx = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::cos(2*M_PI*x2); - vy = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::sin(2*M_PI*x2); - vz = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(y1))*Kokkos::sin(2*M_PI*y2); - - //Compute the relative velocity - auto rel_vx = vx - particle_info.direction[0]*old_mag_v; - auto rel_vy = vy - particle_info.direction[1]*old_mag_v; - auto rel_vz = vz - particle_info.direction[2]*old_mag_v; - auto mag_v2 = rel_vx*rel_vx + rel_vy*rel_vy + rel_vz*rel_vz; - - //Generate random number and compare to sigma/sigma_max - if (rand_gen.drand(0., 1.) < analytic_cross_section_CE(0.5*mp*mag_v2)/max_sigma_cx) { - rejection_test = true; - } + auto old_mag_v = Kokkos::sqrt(2*particle_energy(particle_info.particle_index)/mp); + double vx; + double vy; + double vz; + //Loops this until it passes the rejection test + while (!rejection_test) { + //Generate random numbers for the velocity + double x1 = rand_gen.drand(0., 1.); + double x2 = rand_gen.drand(0., 1.); + double y1 = rand_gen.drand(0., 1.); + double y2 = rand_gen.drand(0., 1.); + + //Sample a new velocity from the thermal distribution (cm/s) + vx = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::cos(2*M_PI*x2); + vy = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::sin(2*M_PI*x2); + vz = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(y1))*Kokkos::sin(2*M_PI*y2); + + //Compute the relative velocity (note using this isn't thoroughly tested. Can be disabled by changing + //old_mag_v to 0, then it is just vx, vy, vz like I originally used. + auto rel_vx = vx - particle_info.direction[0]*old_mag_v; + auto rel_vy = vy - particle_info.direction[1]*old_mag_v; + auto rel_vz = vz - particle_info.direction[2]*old_mag_v; + auto mag_v2 = rel_vx*rel_vx + rel_vy*rel_vy + rel_vz*rel_vz; + + //Generate random number and compare to sigma/sigma_max + if (rand_gen.drand(0., 1.) < analytic_cross_section_CE(0.5*mp*mag_v2)/max_sigma_cx) { + rejection_test = true; } + } - //Now that the rejection test has passed, set the new information - double mag_v{Kokkos::sqrt(vx*vx+vy*vy+vz*vz)}; + //Now that the rejection test has passed, set the new information + double mag_v{Kokkos::sqrt(vx*vx+vy*vy+vz*vz)}; particle_info.direction[0] = vx/mag_v; particle_info.direction[1] = vy/mag_v; particle_info.direction[2] = vz/mag_v; From 231f36c65200c35c430563a465e24bb1bb6138b7 Mon Sep 17 00:00:00 2001 From: Gabe Date: Wed, 23 Jul 2025 14:52:35 -0400 Subject: [PATCH 104/139] Define max_sigma_cx --- src/physics/degas2/DG2Physics.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 24addc9..31be246 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -172,6 +172,7 @@ class DG2Physics { //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 + double max_sigma_cx = 3.8e-14; bool rejection_test = false; auto old_mag_v = Kokkos::sqrt(2*particle_energy(particle_info.particle_index)/mp); From 603dc73d54fa923cbaebbb27e7231e55693891d4 Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Thu, 24 Jul 2025 00:48:38 -0400 Subject: [PATCH 105/139] specular reflection not tested --- src/pumitallyopenmc/pumitally_impl.cpp | 178 ++++++++++++++++++++++++- 1 file changed, 175 insertions(+), 3 deletions(-) diff --git a/src/pumitallyopenmc/pumitally_impl.cpp b/src/pumitallyopenmc/pumitally_impl.cpp index 35c0997..5c9c66d 100644 --- a/src/pumitallyopenmc/pumitally_impl.cpp +++ b/src/pumitallyopenmc/pumitally_impl.cpp @@ -351,6 +351,177 @@ void apply_boundary_condition(Omega_h::Mesh &mesh, PPPS *ptcls, "apply vacumm boundary condition"); } +void compute_boundary_normals(Omega_h::Mesh &mesh) { + const auto exposed_edges = Omega_h::mark_exposed_sides(&mesh); + const auto face2elems = mesh.ask_up(mesh.dim() - 1, mesh.dim()).ab2b; + const auto face2elemsOffset = mesh.ask_up(mesh.dim() - 1, mesh.dim()).a2ab; + const auto elem2nodes = mesh.ask_down(mesh.dim(), 0).ab2b; + const auto face2nodes = mesh.ask_down(mesh.dim() - 1, 0).ab2b; + const auto coords = mesh.coords(); + + Omega_h::Write normals(face2elems.size() * 3, 0.0, + "boundary_normals"); + + // calculate the normals for the exposed edges + auto calculate_normals = OMEGA_H_LAMBDA(const Omega_h::LO &face_id) { + if (exposed_edges[face_id]) { + // get this face's nodes + const auto face_nodes = Omega_h::gather_verts<3>(face2nodes, face_id); + const auto face_coords = + Omega_h::gather_vectors<3, 3>(coords, face_nodes); + const auto normal = Omega_h::cross( + face_coords[1] - face_coords[0], + face_coords[2] - face_coords[0]); // cross product to get normal + + const auto norm = Omega_h::norm(normal); + + // fix direction of the normal: get the fourth node of the element and + // check with it + const auto elem_id = + face2elems[face2elemsOffset[face_id]]; // edge sides only have one + // element + const auto elem_nodes = Omega_h::gather_verts<4>(elem2nodes, elem_id); + int fourth_node = -1; + for (int i = 0; i < 4; ++i) { + if (elem_nodes[i] != face_nodes[0] && elem_nodes[i] != face_nodes[1] && + elem_nodes[i] != face_nodes[2]) { + fourth_node = elem_nodes[i]; + break; + } + } + OMEGA_H_CHECK_PRINTF(fourth_node != -1, + "Error: fourth node not found for face %d\n", + face_id); + + const Omega_h::Vector<3> fourth_node_coord = { + coords[fourth_node * 3 + 0], coords[fourth_node * 3 + 1], + coords[fourth_node * 3 + 2]}; + + // check if the normal points towards the fourth node + Omega_h::Vector<3> fourth_2_face_vector = { + fourth_node_coord[0] - face_coords[0][0], + fourth_node_coord[1] - face_coords[0][1], + fourth_node_coord[2] - face_coords[0][2]}; + + Omega_h::Vector<3> inner_norm; + if (Omega_h::inner_product(normal, fourth_2_face_vector) < 0) { + // flip the normal if it points away from the fourth node + for (int i = 0; i < 3; ++i) { + inner_norm[i] = -normal[i] / norm; + } + } else { + for (int i = 0; i < 3; ++i) { + inner_norm[i] = normal[i] / norm; + } + } + // store the normal in the normals array + normals[face_id * 3 + 0] = inner_norm[0]; + normals[face_id * 3 + 1] = inner_norm[1]; + normals[face_id * 3 + 2] = inner_norm[2]; + } + }; + Omega_h::parallel_for(mesh.nfaces(), calculate_normals, + "compute boundary normals"); + mesh.add_tag(Omega_h::FACE, "normals", 3, Omega_h::Reals(normals)); +} + +void apply_reflection_boundary_condition( + Omega_h::Mesh &mesh, PPPS *ptcls, Omega_h::Write &elem_ids, + Omega_h::Write &next_elems, + Omega_h::Write &ptcl_done, + Omega_h::Write &lastExit, Omega_h::Write &xFace, + Omega_h::Write &inter_points, + Omega_h::Write material_ids, bool initial) { + + // TODO: make this a member variable of the struct + auto particle_destination = ptcls->get<1>(); + auto particle_origin = ptcls->get<0>(); + + const auto class_ids = mesh.get_array(3, "class_id"); + const auto normals = mesh.get_array(Omega_h::FACE, "normals"); + + auto checkExposedEdges = + PS_LAMBDA(const int e, const int pid, const int mask) { + if (mask > 0 && !ptcl_done[pid]) { + bool reached_destination = (lastExit[pid] == -1); + bool hit_outer_boundary = + ((next_elems[pid] == -1) && (elem_ids[pid] != -1)); + + // for the initial run, we need to find the initial position of the + // particles + bool hit_material_boundary = false; + if (!initial) { // stop at geometry boundary + if (next_elems[pid] != -1) { + if (class_ids[elem_ids[pid]] != + class_ids[next_elems[pid]]) { // particle crosses geometry + // boundary + hit_material_boundary = true; + material_ids[pid] = class_ids[next_elems[pid]]; + } + } else { + material_ids[pid] = -1; // no material id if not in an element + } + } + + ptcl_done[pid] = + (reached_destination || hit_material_boundary) ? 1 : ptcl_done[pid]; + // assert that if the next element is -1, then the material id is -1 + if (!initial) { + if (next_elems[pid] == -1) { + OMEGA_H_CHECK_PRINTF( + material_ids[pid] == -1, + "Error: next_elems[%d] is -1 but material_ids[%d] " + "is %d\n", + pid, pid, material_ids[pid]); + } + // printf("Pid %d next element %d, elem id %d, material id %d\n", + // pid, next_elems[pid], elem_ids[pid], material_ids[pid]); + } + + // reflective boundary condition + if (hit_outer_boundary) { // just reached the boundary + // printf("Moving particle %4d (from -> to): element (%5d -> %5d) + // Material (%3d -> %3d)\n", + // pid, elem_ids[pid], next_elems[pid], + // class_ids[elem_ids[pid]], material_ids[pid]); + xFace[pid] = lastExit[pid]; + OMEGA_H_CHECK_PRINTF(lastExit[pid] != -1, + "Error: lastExit[%d] is -1 but " + "hit_outer_boundary is true\n", + pid); + + // change direction + auto normal = Omega_h::Vector<3>{normals[lastExit[pid] * 3 + 0], + normals[lastExit[pid] * 3 + 1], + normals[lastExit[pid] * 3 + 2]}; + Omega_h::Vector<3> particle_direction = { + particle_destination(pid, 0) - particle_origin(pid, 0), + particle_destination(pid, 1) - particle_origin(pid, 1), + particle_destination(pid, 2) - particle_origin(pid, 2)}; + // reflect the particle direction + Omega_h::Vector<3> reflected_direction = + particle_direction - + 2.0 * Omega_h::inner_product(particle_direction, normal) * normal; + + // change the particle's position + // particle reaches the boundary + particle_origin(pid, 0) = inter_points[pid * 3]; + particle_origin(pid, 1) = inter_points[pid * 3 + 1]; + particle_origin(pid, 2) = inter_points[pid * 3 + 2]; + + particle_destination(pid, 0) = + particle_origin(pid, 0) + reflected_direction[0]; + particle_destination(pid, 1) = + particle_origin(pid, 1) + reflected_direction[1]; + particle_destination(pid, 2) = + particle_origin(pid, 2) + reflected_direction[2]; + } + } + }; + pumipic::parallel_for(ptcls, checkExposedEdges, + "apply reflective boundary condition"); +} + void PumiParticleAtElemBoundary::initialize_flux_array(size_t nelems, size_t nEgroups) { Kokkos::resize(flux_, nelems, nEgroups, 3); @@ -385,9 +556,9 @@ void PumiParticleAtElemBoundary::operator()( evaluateFlux(ptcls, inter_points, elem_ids, ptcl_done); updatePrevXPoint(inter_points); } - apply_boundary_condition(mesh, ptcls, elem_ids, next_elems, ptcl_done, - lastExit, inter_faces, inter_points, material_ids_, - initial_); + apply_reflection_boundary_condition(mesh, ptcls, elem_ids, next_elems, + ptcl_done, lastExit, inter_faces, + inter_points, material_ids_, initial_); move_to_next_element(ptcls, elem_ids, next_elems); } @@ -750,6 +921,7 @@ void PumiTallyImpl::load_pumipic_mesh_and_init_particles(int &argc, read_pumipic_lib_and_full_mesh(argc, argv); Omega_h::Mesh *mesh = partition_pumipic_mesh(); create_and_initialize_pumi_particle_structure(mesh); + compute_boundary_normals(*mesh); } } // namespace pumiinopenmc From 5eae2547778edfb305b03e6b25ce7169429c083e Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Thu, 24 Jul 2025 02:42:25 -0400 Subject: [PATCH 106/139] uniform 3d distribution over the whole domain - not tested - will break for openmc (the original initial search is removed) - initial search not needed for uniform --- src/physics/degas2/degas2-main.cpp | 40 ++-- src/pumitallyopenmc/pumitally_impl.cpp | 268 ++++++++++++++++++++++++- src/pumitallyopenmc/pumitally_impl.tpp | 12 +- 3 files changed, 284 insertions(+), 36 deletions(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 93e3707..4d20555 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -13,15 +13,12 @@ // ******************************************* Helper Functions // ******************************************* // -enum class SourceDistribution { - UNIFORM, // Source uniformly distributed across the mesh - EQUAL // Source at centroids of each element -}; struct InputParameters { std::string mesh_name; int num_particles = 0; int max_iterations = 1000; - SourceDistribution source_distribution = SourceDistribution::EQUAL; + pumiinopenmc::SourceDistribution source_distribution = + pumiinopenmc::SourceDistribution::EQUAL; }; void read_input_parameters(int argc, char *const *argv, InputParameters ¶ms); @@ -30,7 +27,6 @@ void print_initial_info(const std::string &mesh_name, // computes centroids of each element in the mesh and stores them in a tag void get_centroids(Omega_h::Mesh &mesh, Omega_h::Write centroids); - template void vector2write(const std::vector &vec, Omega_h::Write &write_vec) { Omega_h::HostWrite host_write_vec(vec.size()); @@ -49,9 +45,9 @@ struct Fields { }; void get_field_values(Omega_h::Reals centroids, Fields &fields); void set_field_values_to_mesh(Omega_h::Mesh &mesh, const Fields &fields); -void set_source_particles(Omega_h::Mesh &mesh, - SourceDistribution source_distribution, - Omega_h::Write particle_positions); +// void set_source_particles(Omega_h::Mesh &mesh, +// SourceDistribution source_distribution, +// Omega_h::Write particle_positions); void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, const Omega_h::Read &electron_density, const Omega_h::Read ion_density, @@ -79,6 +75,7 @@ int main(int argc, char *argv[]) { Fields fields; get_field_values(Omega_h::Reals(centroids), fields); set_field_values_to_mesh(mesh, fields); + pumi_tally.is_pumipic_initialized = true; Kokkos::View sigma_t_; // mat, T, g Kokkos::View sigma_a_; // mat, T, g @@ -88,14 +85,6 @@ int main(int argc, char *argv[]) { sigma_s_); DG2Physics physics(crossSection, input_params.num_particles); - // Initialize particles - Omega_h::Write particle_positions(pumi_tally.pumi_ps_size * 3, - "particle_positions"); - set_source_particles(mesh, input_params.source_distribution, - particle_positions); - pumi_tally.device_pos_buffer_ = particle_positions; - pumi_tally.search_initial_elements(); - // Move particles auto electron_density = mesh.get_array(Omega_h::REGION, "electron_density"); @@ -215,17 +204,23 @@ void initialize_equal_source(const Omega_h::Mesh &mesh, Kokkos::fence(); } +/* void set_source_particles(Omega_h::Mesh &mesh, const SourceDistribution source_distribution, Omega_h::Write particle_positions) { OMEGA_H_CHECK_PRINTF(particle_positions.size() > 0, "Particle positions size (%ld) must be greater than 0\n", particle_positions.size()); + Omega_h::LO num_particles = particle_positions.size() / 3; switch (source_distribution) { - case SourceDistribution::UNIFORM: - throw std::runtime_error("Uniform distribution is not implemented yet."); - break; + case SourceDistribution::UNIFORM: { + auto ppe = pumiinopenmc::PPPS::kkLidView("ptcls_per_element", + mesh.nelems()); + distributeParticlesBasesOnVolume(mesh, ppe, num_particles); + initialize_uniform_source(mesh, particle_positions, ppe); + break; + } case SourceDistribution::EQUAL: { initialize_equal_source(mesh, particle_positions); @@ -235,6 +230,7 @@ void set_source_particles(Omega_h::Mesh &mesh, throw std::runtime_error("Unsupported source distribution!\n"); } } +*/ void set_field_values_to_mesh(Omega_h::Mesh &mesh, const Fields &fields) { OMEGA_H_CHECK_PRINTF(fields.electron_temperature.size() == mesh.nelems(), @@ -355,9 +351,9 @@ void read_input_parameters(int argc, char *const *argv, std::string source_dist_str = argv[4]; // FIXME: hardcoded lower case if (source_dist_str == "uniform") { - params.source_distribution = SourceDistribution::UNIFORM; + params.source_distribution = pumiinopenmc::SourceDistribution::UNIFORM; } else if (source_dist_str == "equal") { - params.source_distribution = SourceDistribution::EQUAL; + params.source_distribution = pumiinopenmc::SourceDistribution::EQUAL; } else { throw std::runtime_error( "Invalid source distribution. Use 'uniform' or 'equal'."); diff --git a/src/pumitallyopenmc/pumitally_impl.cpp b/src/pumitallyopenmc/pumitally_impl.cpp index 5c9c66d..0522cdb 100644 --- a/src/pumitallyopenmc/pumitally_impl.cpp +++ b/src/pumitallyopenmc/pumitally_impl.cpp @@ -10,6 +10,13 @@ namespace pumiinopenmc { +void distributeParticlesBasesOnVolume(Omega_h::Mesh &mesh, + pumiinopenmc::PPPS::kkLidView ppe, + const int numPtcls); +void initialize_uniform_source(Omega_h::Mesh &mesh, + Omega_h::Write particle_positions, + pumiinopenmc::PPPS::kkLidView ppe); + void TallyTimes::print_times() const { printf("\n"); printf("[TIME] Initialization time : %f seconds\n", initialization_time); @@ -20,7 +27,8 @@ void TallyTimes::print_times() const { } PumiTallyImpl::PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, - int &argc, char **&argv) { + int &argc, char **&argv, + SourceDistribution source_dist) { pumi_ps_size = num_particles; oh_mesh_fname = mesh_filename; @@ -33,9 +41,11 @@ PumiTallyImpl::PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, // todo can track lengths be here? - load_pumipic_mesh_and_init_particles(argc, argv); - start_pumi_particles_in_0th_element(*p_picparts_->mesh(), - pumipic_ptcls.get()); + load_pumipic_mesh_and_init_particles(argc, argv, source_dist); + if (source_dist == SourceDistribution::ZERO) { + start_pumi_particles_in_0th_element(*p_picparts_->mesh(), + pumipic_ptcls.get()); + } p_particle_tracer_ = std::make_unique< ParticleTracer>( @@ -359,7 +369,7 @@ void compute_boundary_normals(Omega_h::Mesh &mesh) { const auto face2nodes = mesh.ask_down(mesh.dim() - 1, 0).ab2b; const auto coords = mesh.coords(); - Omega_h::Write normals(face2elems.size() * 3, 0.0, + Omega_h::Write normals(mesh.nfaces() * 3, 0.0, "boundary_normals"); // calculate the normals for the exposed edges @@ -776,6 +786,32 @@ void PumiTallyImpl::search_and_rebuild(bool initial, const bool migrate) { } } +std::unique_ptr +pp_create_particle_structure(Omega_h::Mesh mesh, pumipic::lid_t numPtcls, + pumiinopenmc::PPPS::kkLidView ptcls_per_elem) { + Omega_h::Int ne = mesh.nelems(); + pumiinopenmc::PPPS::kkGidView element_gids("element_gids", ne); + + Kokkos::TeamPolicy policy; + + Omega_h::parallel_for( + ne, OMEGA_H_LAMBDA(const Omega_h::LO &i) { element_gids(i) = i; }); + +#ifdef PUMI_USE_KOKKOS_CUDA + printf("PumiPIC Using GPU for simulation...\n"); + policy = Kokkos::TeamPolicy(10000, 32); +#else + printf("PumiPIC Using CPU for simulation...\n"); + policy = + Kokkos::TeamPolicy(10000, Kokkos::AUTO()); +#endif + + auto ptcls = std::make_unique>( + policy, ne, numPtcls, ptcls_per_elem, element_gids); + + return ptcls; +} + std::unique_ptr pp_create_particle_structure(Omega_h::Mesh mesh, pumipic::lid_t numPtcls) { Omega_h::Int ne = mesh.nelems(); @@ -884,9 +920,35 @@ Omega_h::Mesh *PumiTallyImpl::partition_pumipic_mesh() { } void PumiTallyImpl::create_and_initialize_pumi_particle_structure( - Omega_h::Mesh *mesh) { - pumipic_ptcls = pp_create_particle_structure(*mesh, pumi_ps_size); - start_pumi_particles_in_0th_element(*mesh, pumipic_ptcls.get()); + Omega_h::Mesh *mesh, SourceDistribution source_dist) { + PPPS::kkLidView ppe("ptcls_per_elem", mesh->nelems()); + if (source_dist == SourceDistribution::UNIFORM) { + distributeParticlesBasesOnVolume(*mesh, ppe, pumi_ps_size); + } + pumipic_ptcls = + pp_create_particle_structure(*mesh, pumi_ps_size, ppe); // fixme + if (source_dist == SourceDistribution::UNIFORM) { + auto device_pos_buffer_l = device_pos_buffer_; + initialize_uniform_source(*mesh, device_pos_buffer_l, ppe); + // copy the device positions to the particle structure + auto init_loc = pumipic_ptcls->get<0>(); + auto pids = pumipic_ptcls->get<2>(); + + auto copy_initial_positions = + PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if (mask > 0) { + pids(pid) = pid; + init_loc(pid, 0) = device_pos_buffer_l[pid * 3 + 0]; + init_loc(pid, 1) = device_pos_buffer_l[pid * 3 + 1]; + init_loc(pid, 2) = device_pos_buffer_l[pid * 3 + 2]; + } + }; + pumipic::parallel_for(pumipic_ptcls.get(), copy_initial_positions, + "copy initial positions from device buffer"); + } + if (source_dist == SourceDistribution::ZERO) { + start_pumi_particles_in_0th_element(*mesh, pumipic_ptcls.get()); + } p_pumi_particle_at_elem_boundary_handler = std::make_unique( mesh->nelems(), pumipic_ptcls->capacity()); @@ -916,12 +978,196 @@ void PumiTallyImpl::read_pumipic_lib_and_full_mesh(int &argc, char **&argv) { full_mesh_.nelems()); } -void PumiTallyImpl::load_pumipic_mesh_and_init_particles(int &argc, - char **&argv) { +void PumiTallyImpl::load_pumipic_mesh_and_init_particles( + int &argc, char **&argv, SourceDistribution source_dist) { read_pumipic_lib_and_full_mesh(argc, argv); Omega_h::Mesh *mesh = partition_pumipic_mesh(); - create_and_initialize_pumi_particle_structure(mesh); + create_and_initialize_pumi_particle_structure(mesh, source_dist); compute_boundary_normals(*mesh); } +OMEGA_H_DEVICE o::Real volume_tet(const o::Few, 4> &tet_verts) { + o::Few, 3> basis33 = {tet_verts[1] - tet_verts[0], + tet_verts[2] - tet_verts[0], + tet_verts[3] - tet_verts[0]}; + auto volume = o::tet_volume_from_basis(basis33); + return volume; +} + +o::Real volume_of_3d_mesh(o::Mesh &mesh) { + OMEGA_H_CHECK_PRINTF(mesh.dim() == 3, + "Volume calculation is only supported for 3D meshes, " + "but got %dD mesh\n", + mesh.dim()); + const auto coords = mesh.coords(); + const auto elems2nodes = mesh.ask_down(o::REGION, o::VERT).ab2b; + const auto n_elems = mesh.nelems(); + o::Real total_volume = 0.0; + + Kokkos::parallel_reduce( + n_elems, + KOKKOS_LAMBDA(const int i, o::Real &local_volume) { + auto elem_nodes = o::gather_verts<4>(elems2nodes, i); + o::Few, 4> elem_coords; + elem_coords = o::gather_vectors<4, 3>(coords, elem_nodes); + o::Real elem_volume = volume_tet(elem_coords); + local_volume += elem_volume; + }, + Kokkos::Sum(total_volume)); + + return total_volume; +} + +void distributeParticlesBasesOnVolume(Omega_h::Mesh &mesh, + pumiinopenmc::PPPS::kkLidView ppe, + const int numPtcls) { + OMEGA_H_CHECK_PRINTF(mesh.dim() == 3, + "Distributing particles based on volume is only " + "supported for 3D meshes, but got %dD mesh\n", + mesh.dim()); + o::LO ne = mesh.nelems(); + o::Real mesh_volume = volume_of_3d_mesh(mesh); + OMEGA_H_CHECK(mesh_volume > 0.0); + + auto coords = mesh.coords(); + auto element2nodes = mesh.ask_down(o::REGION, o::VERT).ab2b; + + auto distribute_based_on_volume = OMEGA_H_LAMBDA(o::LO e) { + auto verts = o::gather_verts<4>(element2nodes, e); + auto vert_coords = o::gather_vectors<4, 3>(coords, verts); + o::Real vol = volume_tet(vert_coords); +#ifdef DEBUG + OMEGA_H_CHECK(area > 0.0); +#endif + o::Real volume_fraction = vol / mesh_volume; + ppe[e] = std::round(numPtcls * volume_fraction); + }; + o::parallel_for(ne, distribute_based_on_volume); + + Omega_h::LO totPtcls = 0; + Kokkos::parallel_reduce( + ppe.size(), + OMEGA_H_LAMBDA(const int i, Omega_h::LO &lsum) { lsum += ppe[i]; }, + totPtcls); + + // remove or add particles to match the total number of particles + int extra_particles = numPtcls - totPtcls; + // go throught the first extra_particles elements and add/remove one particle + OMEGA_H_CHECK_PRINTF(extra_particles <= mesh.nelems(), + "Extra particles (%d) should be less than or equal to " + "number of elements (%d)\n", + extra_particles, mesh.nelems()); + + int add_remove = (extra_particles > 0) ? 1 : -1; + auto add_or_remove_particles = OMEGA_H_LAMBDA(o::LO e) { + ppe[e] += add_remove; + }; + o::parallel_for(std::abs(extra_particles), add_or_remove_particles); +} + +OMEGA_H_DEVICE o::Few, 3> +barycentric_basis(const o::Few, 3> &tri_verts) { + o::Few, 3> basis; + for (int i = 0; i < 3; i++) { + basis[0][i] = tri_verts[i][0]; + basis[1][i] = tri_verts[i][1]; + basis[2][i] = 1.0; + } + return basis; +} + +OMEGA_H_DEVICE o::Few, 4> +barycentric_basis(const o::Few, 4> &tet_verts) { + o::Few, 4> basis; + for (int i = 0; i < 4; ++i) { + basis[0][i] = tet_verts[i][0]; + basis[1][i] = tet_verts[i][1]; + basis[2][i] = tet_verts[i][2]; + basis[3][i] = 1.0; + } + return basis; +} + +OMEGA_H_DEVICE o::Vector<3> +barycentric2real(const o::Few, 4> &tet_verts, + const o::Vector<4> &bary) { + o::Few, 4> basis = barycentric_basis(tet_verts); + o::Vector<3> real_coords; + // real_coords = basis * bary; + real_coords[0] = basis[0][0] * bary[0] + basis[1][0] * bary[1] + + basis[2][0] * bary[2] + basis[3][0] * bary[3]; + real_coords[1] = basis[0][1] * bary[0] + basis[1][1] * bary[1] + + basis[2][1] * bary[2] + basis[3][1] * bary[3]; + real_coords[2] = basis[0][2] * bary[0] + basis[1][2] * bary[1] + + basis[2][2] * bary[2] + basis[3][2] * bary[3]; + + return real_coords; +} + +void initialize_uniform_source(Omega_h::Mesh &mesh, + Omega_h::Write particle_positions, + pumiinopenmc::PPPS::kkLidView ppe) { + int dim = mesh.dim(); + OMEGA_H_CHECK(dim == 3); + + // cumulative sum of particles per element + Omega_h::Write cumulative_particles(mesh.nelems() + 1, 0); + Omega_h::LO num_particles_cumsum = 0; + auto calculate_cumulative_number_of_particles = + KOKKOS_LAMBDA(const int &e, Omega_h::LO &cumulative, bool is_final) { + auto num_particles = ppe[e]; + cumulative += num_particles; + if (is_final) { + cumulative_particles[e + 1] = cumulative; + } + }; + Kokkos::parallel_scan("calculate_cumulative_number_of_particles", + mesh.nelems(), calculate_cumulative_number_of_particles, + num_particles_cumsum); + + OMEGA_H_CHECK_PRINTF(num_particles_cumsum == particle_positions.size() / 3, + "Total number of particles (%ld) does not match " + "cumulative particles (%ld)\n", + particle_positions.size() / 3, num_particles_cumsum); + + const auto cells2nodes = mesh.ask_down(o::REGION, o::VERT).ab2b; + const auto coords = mesh.coords(); + + Kokkos::Random_XorShift64_Pool random_pool; + auto set_initial_positions = OMEGA_H_LAMBDA(const int &e) { + auto pid_start = cumulative_particles[e]; + auto pid_end = cumulative_particles[e + 1]; + auto num_particles_in_element = pid_end - pid_start; + OMEGA_H_CHECK(num_particles_in_element >= 0); + + for (Omega_h::LO pid = pid_start; pid < pid_end; ++pid) { + auto gen = random_pool.get_state(); + o::Vector<4> random_bcc{0.0, 0.0, 0.0, 0.0}; + random_bcc[0] = gen.drand(0.0, 1.0); + random_bcc[1] = gen.drand(0.0, 1.0); + random_bcc[2] = gen.drand(0.0, 1.0); + o::Real complimentary0 = 1.0 - random_bcc[0]; + o::Real complimentary1 = 1.0 - random_bcc[1]; + o::Real complimentary2 = 1.0 - random_bcc[2]; + + bool more_than_one = random_bcc[0] + random_bcc[1] + random_bcc[2] > 1.0; + random_bcc[0] = more_than_one ? complimentary0 : random_bcc[0]; + random_bcc[1] = more_than_one ? complimentary1 : random_bcc[1]; + random_bcc[2] = more_than_one ? complimentary2 : random_bcc[2]; + random_bcc[2] = 1.0 - random_bcc[0] - random_bcc[1] - random_bcc[2]; + random_pool.free_state(gen); + + auto verts = o::gather_verts<4>(cells2nodes, e); + auto vert_coords = o::gather_vectors<4, 3>(coords, verts); + + auto real_loc = barycentric2real(vert_coords, random_bcc); + + particle_positions[pid * 3 + 0] = real_loc[0]; + particle_positions[pid * 3 + 1] = real_loc[1]; + particle_positions[pid * 3 + 2] = real_loc[2]; + } + }; + o::parallel_for(mesh.nelems(), set_initial_positions); +} + } // namespace pumiinopenmc diff --git a/src/pumitallyopenmc/pumitally_impl.tpp b/src/pumitallyopenmc/pumitally_impl.tpp index fc693c7..f3dd5a8 100644 --- a/src/pumitallyopenmc/pumitally_impl.tpp +++ b/src/pumitallyopenmc/pumitally_impl.tpp @@ -20,6 +20,12 @@ struct TallyTimes { void print_times() const; }; +enum class SourceDistribution { + UNIFORM, // Source uniformly distributed across the mesh + EQUAL, // Source at centroids of each element + ZERO // in the zeroth element centroid +}; + // ------------------------------------------------------------------------------------------------// // * Data structure for PumiPic // Particle: 0-origin, 1-destination, 2-particle_id, @@ -123,15 +129,15 @@ public: // * Constructor PumiTallyImpl(std::string &mesh_filename, int64_t num_particles, int &argc, - char **&argv); // fixme extra & + char **&argv, SourceDistribution source_dist = SourceDistribution::ZERO); // fixme extra & // * Destructor ~PumiTallyImpl() = default; // Functions - void create_and_initialize_pumi_particle_structure(Omega_h::Mesh *mesh); + void create_and_initialize_pumi_particle_structure(Omega_h::Mesh *mesh, SourceDistribution source_dist); - void load_pumipic_mesh_and_init_particles(int &argc, char **&argv); + void load_pumipic_mesh_and_init_particles(int &argc, char **&argv, SourceDistribution source_dist); Omega_h::Mesh *partition_pumipic_mesh(); From 79dd7f628461649d80feea4eff35139d2673a1a7 Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Thu, 24 Jul 2025 02:48:06 -0400 Subject: [PATCH 107/139] Revert "Degas2 physics Field Value Inputting" --- src/physics/degas2/DG2Physics.h | 54 +++++++----------------------- src/physics/degas2/degas2-main.cpp | 37 -------------------- test/test_degas2_physics.cpp | 5 ++- 3 files changed, 14 insertions(+), 82 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 31be246..fb47cd8 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -20,7 +20,6 @@ struct ParticleInfo { double weight; int energy_group; // Energy group *index* int particle_index; - double alpha; //To be multiplied in when computing tally }; struct FieldInfo { @@ -134,10 +133,6 @@ class DG2Physics { return sigma_cx; } - KOKKOS_FUNCTION //IF THIS IS CHANGED MAY NEED TO CHANGE max_sigma_cx in the collision code - double analytic_cross_section_CE(double energy) const{ - return (0.6937e-14*(1-0.155*Kokkos::log10(energy))*(1-0.155*Kokkos::log10(energy)))/(1+0.1112e-14*Kokkos::pow(energy,3.3)); - } KOKKOS_FUNCTION void sample_collision_distance(ParticleInfo &particle_info, @@ -164,49 +159,26 @@ class DG2Physics { //Generate Random Numbers auto rand_gen = random_pool.get_state(); + double x1 = rand_gen.drand(0., 1.); + double x2 = rand_gen.drand(0., 1.); + double y1 = rand_gen.drand(0., 1.); + double y2 = rand_gen.drand(0., 1.); double ww = rand_gen.drand(0., 1.); + random_pool.free_state(rand_gen); double sigma_ion = ionization_cross_section(particle_info, field_info); double sigma_cx = charge_exchange_cross_section(particle_info, field_info); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) - double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double max_sigma_cx = 3.8e-14; - - bool rejection_test = false; - auto old_mag_v = Kokkos::sqrt(2*particle_energy(particle_info.particle_index)/mp); - double vx; - double vy; - double vz; - //Loops this until it passes the rejection test - while (!rejection_test) { - //Generate random numbers for the velocity - double x1 = rand_gen.drand(0., 1.); - double x2 = rand_gen.drand(0., 1.); - double y1 = rand_gen.drand(0., 1.); - double y2 = rand_gen.drand(0., 1.); - - //Sample a new velocity from the thermal distribution (cm/s) - vx = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::cos(2*M_PI*x2); - vy = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::sin(2*M_PI*x2); - vz = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(y1))*Kokkos::sin(2*M_PI*y2); - - //Compute the relative velocity (note using this isn't thoroughly tested. Can be disabled by changing - //old_mag_v to 0, then it is just vx, vy, vz like I originally used. - auto rel_vx = vx - particle_info.direction[0]*old_mag_v; - auto rel_vy = vy - particle_info.direction[1]*old_mag_v; - auto rel_vz = vz - particle_info.direction[2]*old_mag_v; - auto mag_v2 = rel_vx*rel_vx + rel_vy*rel_vy + rel_vz*rel_vz; - - //Generate random number and compare to sigma/sigma_max - if (rand_gen.drand(0., 1.) < analytic_cross_section_CE(0.5*mp*mag_v2)/max_sigma_cx) { - rejection_test = true; - } - } + double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 + + auto vx = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::cos(2*M_PI*x2); + auto vy = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::sin(2*M_PI*x2); + auto vz = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(y1))*Kokkos::sin(2*M_PI*y2); + + auto mag_v = Kokkos::sqrt(vx*vx + vy*vy + vz*vz); - //Now that the rejection test has passed, set the new information - double mag_v{Kokkos::sqrt(vx*vx+vy*vy+vz*vz)}; particle_info.direction[0] = vx/mag_v; particle_info.direction[1] = vy/mag_v; particle_info.direction[2] = vz/mag_v; @@ -231,8 +203,6 @@ class DG2Physics { } } particle_info.weight = new_weight; - random_pool.free_state(rand_gen); - } //To here diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 419ca59..93e3707 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -7,8 +7,6 @@ #include #include #include -#include -#include #include "DG2Physics.h" #include "pumitally_impl.tpp" @@ -313,9 +311,6 @@ void get_centroids(Omega_h::Mesh &mesh, } // TODO: Implement the function to retrieve field values based on centroids -// NOTE: This requires the python script and the plasma source mesh and data -// to be in the folder with this cpp file. The python script may need to be -// updated to read in different plasma source data. void get_field_values(Omega_h::Reals centroids, Fields &fields) { auto centroids_h = Omega_h::HostRead(centroids); @@ -331,38 +326,6 @@ void get_field_values(Omega_h::Reals centroids, Fields &fields) { fields.bulk_flow_velocity.resize(num_elements, 0.0); // TODO: Add your code here to retrieve the actual field values - // Write centroid coords to CSV to be read by python script - std::ofstream coords("centroid_coords.csv"); - for (int j = 0; j < (centroids_h.size() / 3); j++) { - coords << j << "," << centroids_h[j*3 + 0] << "," << centroids_h[j*3 + 1] << "," << centroids_h[j*3 + 2] << std::endl; - } - coords.close(); - - // Run the python script - system("python Mesh_Map.py"); - - // Read in the field values - std::ifstream fieldvals{"vals.csv"}; - int i = 0; - std::string IDString; - std::string xxString; - std::string yyString; - std::string zzString; - - while (std::getline(fieldvals, IDString, ',')) { - - std::getline(fieldvals,xxString, ','); - std::getline(fieldvals,yyString, ','); - std::getline(fieldvals,zzString, '\n'); - - // Asign field values - fields.ion_density[i] = std::stod(xxString); - fields.electron_density[i] = std::stod(xxString); - fields.electron_temperature[i] = std::stod(yyString); - fields.ion_temperature[i] = std::stod(zzString); - - i++; - } } void print_initial_info(const std::string &mesh_name, diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 28154e1..9279c88 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -38,7 +38,6 @@ TEST_CASE("Test Degas2 Physics Functions"){ particles(i).energy_group = energy; particles(i).weight = 1.0; particles(i).particle_index = i; - particles(i).alpha = 1.0; fields(i).electron_density = 1.0e13; fields(i).ion_density = 1.0e13; @@ -112,8 +111,8 @@ TEST_CASE("Test Degas2 Physics Functions"){ outfile << output(i).direction[2] << std::endl; } */ - REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.04)); - REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.04)); + REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.03)); + REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); } From 13d0bef1d0d88bf683d12632e9522ad123b24468 Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Thu, 24 Jul 2025 02:59:07 -0400 Subject: [PATCH 108/139] Reapply "Degas2 physics Field Value Inputting" This reverts commit 79dd7f628461649d80feea4eff35139d2673a1a7. --- src/physics/degas2/DG2Physics.h | 54 +++++++++++++++++++++++------- src/physics/degas2/degas2-main.cpp | 37 ++++++++++++++++++++ test/test_degas2_physics.cpp | 5 +-- 3 files changed, 82 insertions(+), 14 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index fb47cd8..31be246 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -20,6 +20,7 @@ struct ParticleInfo { double weight; int energy_group; // Energy group *index* int particle_index; + double alpha; //To be multiplied in when computing tally }; struct FieldInfo { @@ -133,6 +134,10 @@ class DG2Physics { return sigma_cx; } + KOKKOS_FUNCTION //IF THIS IS CHANGED MAY NEED TO CHANGE max_sigma_cx in the collision code + double analytic_cross_section_CE(double energy) const{ + return (0.6937e-14*(1-0.155*Kokkos::log10(energy))*(1-0.155*Kokkos::log10(energy)))/(1+0.1112e-14*Kokkos::pow(energy,3.3)); + } KOKKOS_FUNCTION void sample_collision_distance(ParticleInfo &particle_info, @@ -159,26 +164,49 @@ class DG2Physics { //Generate Random Numbers auto rand_gen = random_pool.get_state(); - double x1 = rand_gen.drand(0., 1.); - double x2 = rand_gen.drand(0., 1.); - double y1 = rand_gen.drand(0., 1.); - double y2 = rand_gen.drand(0., 1.); double ww = rand_gen.drand(0., 1.); - random_pool.free_state(rand_gen); double sigma_ion = ionization_cross_section(particle_info, field_info); double sigma_cx = charge_exchange_cross_section(particle_info, field_info); //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) - double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - - auto vx = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::cos(2*M_PI*x2); - auto vy = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::sin(2*M_PI*x2); - auto vz = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(y1))*Kokkos::sin(2*M_PI*y2); - - auto mag_v = Kokkos::sqrt(vx*vx + vy*vy + vz*vz); + double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 + double max_sigma_cx = 3.8e-14; + + bool rejection_test = false; + auto old_mag_v = Kokkos::sqrt(2*particle_energy(particle_info.particle_index)/mp); + double vx; + double vy; + double vz; + //Loops this until it passes the rejection test + while (!rejection_test) { + //Generate random numbers for the velocity + double x1 = rand_gen.drand(0., 1.); + double x2 = rand_gen.drand(0., 1.); + double y1 = rand_gen.drand(0., 1.); + double y2 = rand_gen.drand(0., 1.); + + //Sample a new velocity from the thermal distribution (cm/s) + vx = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::cos(2*M_PI*x2); + vy = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::sin(2*M_PI*x2); + vz = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(y1))*Kokkos::sin(2*M_PI*y2); + + //Compute the relative velocity (note using this isn't thoroughly tested. Can be disabled by changing + //old_mag_v to 0, then it is just vx, vy, vz like I originally used. + auto rel_vx = vx - particle_info.direction[0]*old_mag_v; + auto rel_vy = vy - particle_info.direction[1]*old_mag_v; + auto rel_vz = vz - particle_info.direction[2]*old_mag_v; + auto mag_v2 = rel_vx*rel_vx + rel_vy*rel_vy + rel_vz*rel_vz; + + //Generate random number and compare to sigma/sigma_max + if (rand_gen.drand(0., 1.) < analytic_cross_section_CE(0.5*mp*mag_v2)/max_sigma_cx) { + rejection_test = true; + } + } + //Now that the rejection test has passed, set the new information + double mag_v{Kokkos::sqrt(vx*vx+vy*vy+vz*vz)}; particle_info.direction[0] = vx/mag_v; particle_info.direction[1] = vy/mag_v; particle_info.direction[2] = vz/mag_v; @@ -203,6 +231,8 @@ class DG2Physics { } } particle_info.weight = new_weight; + random_pool.free_state(rand_gen); + } //To here diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 4d20555..9d97bf5 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include "DG2Physics.h" #include "pumitally_impl.tpp" @@ -307,6 +309,9 @@ void get_centroids(Omega_h::Mesh &mesh, } // TODO: Implement the function to retrieve field values based on centroids +// NOTE: This requires the python script and the plasma source mesh and data +// to be in the folder with this cpp file. The python script may need to be +// updated to read in different plasma source data. void get_field_values(Omega_h::Reals centroids, Fields &fields) { auto centroids_h = Omega_h::HostRead(centroids); @@ -322,6 +327,38 @@ void get_field_values(Omega_h::Reals centroids, Fields &fields) { fields.bulk_flow_velocity.resize(num_elements, 0.0); // TODO: Add your code here to retrieve the actual field values + // Write centroid coords to CSV to be read by python script + std::ofstream coords("centroid_coords.csv"); + for (int j = 0; j < (centroids_h.size() / 3); j++) { + coords << j << "," << centroids_h[j*3 + 0] << "," << centroids_h[j*3 + 1] << "," << centroids_h[j*3 + 2] << std::endl; + } + coords.close(); + + // Run the python script + system("python Mesh_Map.py"); + + // Read in the field values + std::ifstream fieldvals{"vals.csv"}; + int i = 0; + std::string IDString; + std::string xxString; + std::string yyString; + std::string zzString; + + while (std::getline(fieldvals, IDString, ',')) { + + std::getline(fieldvals,xxString, ','); + std::getline(fieldvals,yyString, ','); + std::getline(fieldvals,zzString, '\n'); + + // Asign field values + fields.ion_density[i] = std::stod(xxString); + fields.electron_density[i] = std::stod(xxString); + fields.electron_temperature[i] = std::stod(yyString); + fields.ion_temperature[i] = std::stod(zzString); + + i++; + } } void print_initial_info(const std::string &mesh_name, diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 9279c88..28154e1 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -38,6 +38,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ particles(i).energy_group = energy; particles(i).weight = 1.0; particles(i).particle_index = i; + particles(i).alpha = 1.0; fields(i).electron_density = 1.0e13; fields(i).ion_density = 1.0e13; @@ -111,8 +112,8 @@ TEST_CASE("Test Degas2 Physics Functions"){ outfile << output(i).direction[2] << std::endl; } */ - REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.03)); - REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.03)); + REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.04)); + REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.04)); REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); } From 48e7aafd9e25a1835de61c04323dce39ff8477d4 Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Thu, 24 Jul 2025 03:25:34 -0400 Subject: [PATCH 109/139] sample initial energy --- src/physics/degas2/degas2-main.cpp | 77 +++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 9d97bf5..fd554fa 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -4,11 +4,11 @@ * PUMI-Tally interface */ +#include +#include #include #include #include -#include -#include #include "DG2Physics.h" #include "pumitally_impl.tpp" @@ -29,6 +29,8 @@ void print_initial_info(const std::string &mesh_name, // computes centroids of each element in the mesh and stores them in a tag void get_centroids(Omega_h::Mesh &mesh, Omega_h::Write centroids); +void sample_initial_particle_energy(Kokkos::View energy_array); + template void vector2write(const std::vector &vec, Omega_h::Write &write_vec) { Omega_h::HostWrite host_write_vec(vec.size()); @@ -86,6 +88,7 @@ int main(int argc, char *argv[]) { DG2CrossSection crossSection(sigma_t_, sigma_a_, scattering_matrix_, sigma_s_); DG2Physics physics(crossSection, input_params.num_particles); + sample_initial_particle_energy(physics.particle_energy); // Move particles auto electron_density = @@ -330,7 +333,9 @@ void get_field_values(Omega_h::Reals centroids, Fields &fields) { // Write centroid coords to CSV to be read by python script std::ofstream coords("centroid_coords.csv"); for (int j = 0; j < (centroids_h.size() / 3); j++) { - coords << j << "," << centroids_h[j*3 + 0] << "," << centroids_h[j*3 + 1] << "," << centroids_h[j*3 + 2] << std::endl; + coords << j << "," << centroids_h[j * 3 + 0] << "," + << centroids_h[j * 3 + 1] << "," << centroids_h[j * 3 + 2] + << std::endl; } coords.close(); @@ -347,9 +352,9 @@ void get_field_values(Omega_h::Reals centroids, Fields &fields) { while (std::getline(fieldvals, IDString, ',')) { - std::getline(fieldvals,xxString, ','); - std::getline(fieldvals,yyString, ','); - std::getline(fieldvals,zzString, '\n'); + std::getline(fieldvals, xxString, ','); + std::getline(fieldvals, yyString, ','); + std::getline(fieldvals, zzString, '\n'); // Asign field values fields.ion_density[i] = std::stod(xxString); @@ -396,3 +401,63 @@ void read_input_parameters(int argc, char *const *argv, "Invalid source distribution. Use 'uniform' or 'equal'."); } } + +void sample_initial_particle_energy(Kokkos::View energy_array) { + random_pool_t randomPool; + + auto sample_energy = OMEGA_H_LAMBDA(int i) { + // Basically, feed this 4 uniformly generated random numbers, x1, x2, y1, + // y2, on the interval (0,1) and it + // will give you a velocity vector (vx,vy,vz), a unit direction vector + // (directionx, directiony, directionz), the alpha value used in the tally, + // and the energy of the particle that was sampled. All from a gas at + // temperature temp. + + double mp{938.27e6 / (3e10 * 3e10)}; // eV/c^2 = eV*s^2/cm^2. Necessary + // constant for the distribution + + // This is what we set as the source temperature. This could in principle be + // a parameter but for now can be hard coded in as 3 eV or so and I can + // always tweak it. + double temp = 3; // eV. + + // This uses the Box-Muller method of sampling a Gaussian, which generates + // two independent normally distributed values from two independent + // uniformly distributed numbers on the interval (0,1). (x1, x2) is the + // first pair. (y1, y2) is the second pair. Note this technically could make + //4 independent normally distributed values, but we only need three. This + // generates a velocity vector (vx,vy,vz) sampled from an ideal gas at + // temperature temp. + auto generator = randomPool.get_state(); + auto x1 = generator.drand(0.0, 1.0); + auto x2 = generator.drand(0.0, 1.0); + auto y1 = generator.drand(0.0, 1.0); + auto y2 = generator.drand(0.0, 1.0); + randomPool.free_state(generator); + auto vx = Kokkos::sqrt(temp / mp) * Kokkos::sqrt(-2 * Kokkos::log(x1)) * + Kokkos::cos(2 * M_PI * x2); + auto vy = Kokkos::sqrt(temp / mp) * Kokkos::sqrt(-2 * Kokkos::log(x1)) * + Kokkos::sin(2 * M_PI * x2); + auto vz = Kokkos::sqrt(temp / mp) * Kokkos::sqrt(-2 * Kokkos::log(y1)) * + Kokkos::sin(2 * M_PI * y2); + + // Compute the magnitude of the velocity vector + auto mag_v = Kokkos::sqrt(vx * vx + vy * vy + vz * vz); + + // Compute the alpha factor multiplied in when computing the tally + double alpha = 1 / mag_v; + + // Compute the unit normal + double directionx = vx / mag_v; + double directiony = vy / mag_v; + double directionz = vz / mag_v; + + // Compute the particle energy. This you will want to save probably to pass + // to my initialization (set_energy) code so that I have the energy for my + // computations. + double particle_energy = 0.5 * mp * mag_v * mag_v; + energy_array(i) = particle_energy; + }; + Kokkos::parallel_for("sample_initial_particle_energy", energy_array.size(), + sample_energy); +} From c3e36b84b4452ac2dfda776447123e566752fd0a Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Thu, 24 Jul 2025 03:38:34 -0400 Subject: [PATCH 110/139] collide or sample new distance based on if reached destination or not --- src/physics/degas2/degas2-main.cpp | 9 +++++++-- src/pumitallyopenmc/pumitally_impl.cpp | 1 + src/pumitallyopenmc/pumitally_impl.tpp | 5 +++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index fd554fa..f1ac624 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -132,6 +132,9 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, auto particle_orig = pumi_tally.pumipic_ptcls->get<0>(); auto particle_weight = pumi_tally.pumipic_ptcls->get<4>(); auto particle_group = pumi_tally.pumipic_ptcls->get<5>(); + + auto last_exit = + pumi_tally.p_pumi_particle_at_elem_boundary_handler->last_exit_; auto get_new_destination = PS_LAMBDA(const int &e, const int &pid, const int &mask) { if (mask > 0) { // FIXME: check if the particle is at destination or at @@ -160,7 +163,9 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, field_info.bulk_flow_velocity[1] = bulk_flow_velocity[e * 3 + 1]; field_info.bulk_flow_velocity[2] = bulk_flow_velocity[e * 3 + 2]; - physics.collide_particle(particle_info, field_info); + if (last_exit[pid] == -1) { // reached destination + physics.collide_particle(particle_info, field_info); + } physics.sample_collision_distance(particle_info, field_info); // Update particle position and direction @@ -425,7 +430,7 @@ void sample_initial_particle_energy(Kokkos::View energy_array) { // two independent normally distributed values from two independent // uniformly distributed numbers on the interval (0,1). (x1, x2) is the // first pair. (y1, y2) is the second pair. Note this technically could make - //4 independent normally distributed values, but we only need three. This + // 4 independent normally distributed values, but we only need three. This // generates a velocity vector (vx,vy,vz) sampled from an ideal gas at // temperature temp. auto generator = randomPool.get_state(); diff --git a/src/pumitallyopenmc/pumitally_impl.cpp b/src/pumitallyopenmc/pumitally_impl.cpp index 0522cdb..29f2cb9 100644 --- a/src/pumitallyopenmc/pumitally_impl.cpp +++ b/src/pumitallyopenmc/pumitally_impl.cpp @@ -565,6 +565,7 @@ void PumiParticleAtElemBoundary::operator()( if (!initial_) { evaluateFlux(ptcls, inter_points, elem_ids, ptcl_done); updatePrevXPoint(inter_points); + updateLastExit(lastExit); } apply_reflection_boundary_condition(mesh, ptcls, elem_ids, next_elems, ptcl_done, lastExit, inter_faces, diff --git a/src/pumitallyopenmc/pumitally_impl.tpp b/src/pumitallyopenmc/pumitally_impl.tpp index f3dd5a8..062a9c9 100644 --- a/src/pumitallyopenmc/pumitally_impl.tpp +++ b/src/pumitallyopenmc/pumitally_impl.tpp @@ -70,6 +70,10 @@ public: void updatePrevXPoint(Omega_h::Write &xpoints); + void updateLastExit(Omega_h::Write &lastExit) { + last_exit_ = lastExit; + } + void updatePrevXPoint(PPPS *ptcls); void evaluateFlux(PPPS *ptcls, Omega_h::Write xpoints, @@ -92,6 +96,7 @@ public: Omega_h::Write prev_xpoint_; Omega_h::Write total_tracklength_; Omega_h::Write material_ids_; + Omega_h::Write last_exit_; }; // ------------------------------------------------------------------------------------------------// From 8bdb038982f29e44b8e6f879e0871e83e1992e6c Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Thu, 24 Jul 2025 03:50:16 -0400 Subject: [PATCH 111/139] multiply contribution with alpha factor --- src/physics/degas2/degas2-main.cpp | 3 +++ src/pumitallyopenmc/pumitally_impl.cpp | 5 +++-- src/pumitallyopenmc/pumitally_impl.tpp | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index f1ac624..eaf45d3 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -135,6 +135,7 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, auto last_exit = pumi_tally.p_pumi_particle_at_elem_boundary_handler->last_exit_; + auto alpha = pumi_tally.p_pumi_particle_at_elem_boundary_handler->alpha_; auto get_new_destination = PS_LAMBDA(const int &e, const int &pid, const int &mask) { if (mask > 0) { // FIXME: check if the particle is at destination or at @@ -174,6 +175,8 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, particle_dest(pid, 2) = particle_info.position[2]; particle_weight(pid) = particle_info.weight; particle_group(pid) = particle_info.energy_group; + + alpha[pid] = particle_info.alpha; } }; pumipic::parallel_for(pumi_tally.pumipic_ptcls.get(), get_new_destination, diff --git a/src/pumitallyopenmc/pumitally_impl.cpp b/src/pumitallyopenmc/pumitally_impl.cpp index 29f2cb9..9aae074 100644 --- a/src/pumitallyopenmc/pumitally_impl.cpp +++ b/src/pumitallyopenmc/pumitally_impl.cpp @@ -545,7 +545,8 @@ PumiParticleAtElemBoundary::PumiParticleAtElemBoundary(size_t nelems, size_t capacity) : prev_xpoint_(capacity * 3, 0.0, "prev_xpoint"), material_ids_(capacity, -1, "material_ids"), - total_tracklength_(capacity, 0.0, "total_tracklength"), initial_(true) { + total_tracklength_(capacity, 0.0, "total_tracklength"), initial_(true), + alpha_(capacity, 0.0, "alpha") { initialize_flux_array(nelems, 2); // FIXME: hardcoded 2 groups printf("[INFO] Particle handler at boundary with %d elements and %d " "x points size (3 * n_particles)\n", @@ -647,7 +648,7 @@ void PumiParticleAtElemBoundary::evaluateFlux( orig[0], orig[1], orig[2], dest[0], dest[1], dest[2]); } - Omega_h::Real contribution = segment_length * p_wgt(pid); + Omega_h::Real contribution = segment_length * p_wgt(pid) * alpha_[pid]; int group = p_groups(pid); OMEGA_H_CHECK_PRINTF( diff --git a/src/pumitallyopenmc/pumitally_impl.tpp b/src/pumitallyopenmc/pumitally_impl.tpp index 062a9c9..e516a62 100644 --- a/src/pumitallyopenmc/pumitally_impl.tpp +++ b/src/pumitallyopenmc/pumitally_impl.tpp @@ -97,6 +97,7 @@ public: Omega_h::Write total_tracklength_; Omega_h::Write material_ids_; Omega_h::Write last_exit_; + Omega_h::Write alpha_; }; // ------------------------------------------------------------------------------------------------// From baee04637411bf351f796168f013363fc6e6afa9 Mon Sep 17 00:00:00 2001 From: gpaynter5004 Date: Thu, 24 Jul 2025 16:10:12 -0400 Subject: [PATCH 112/139] Update degas2-main.cpp Fix bulk_flow_velocity size --- src/physics/degas2/degas2-main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index eaf45d3..76f61b8 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -335,7 +335,7 @@ void get_field_values(Omega_h::Reals centroids, Fields &fields) { fields.ion_temperature.resize(num_elements, 1.0); fields.electron_density.resize(num_elements, 1.0); fields.ion_density.resize(num_elements, 1.0); - fields.bulk_flow_velocity.resize(num_elements, 0.0); + fields.bulk_flow_velocity.resize(3*num_elements, 0.0); // TODO: Add your code here to retrieve the actual field values // Write centroid coords to CSV to be read by python script From c7fc5e193bc928bc6ecfaa57b551f3b25c95d117 Mon Sep 17 00:00:00 2001 From: Gabe Date: Fri, 25 Jul 2025 12:10:04 -0400 Subject: [PATCH 113/139] Set the alpha to actually update --- src/physics/degas2/DG2Physics.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 31be246..71374d7 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -155,6 +155,9 @@ class DG2Physics { particle_info.position[0] += l*particle_info.direction[0]; particle_info.position[1] += l*particle_info.direction[1]; particle_info.position[2] += l*particle_info.direction[2]; + + double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 + particle_info.alpha = Kokkos::sqrt(mp/(2.0*particle_energy(particle_info.particle_index))); } // collision event @@ -231,6 +234,9 @@ class DG2Physics { } } particle_info.weight = new_weight; + + //This definition may be redundant since it is also in the next location function + particle_info.alpha = Kokkos::sqrt(mp/(2.0*particle_energy(particle_info.particle_index))); random_pool.free_state(rand_gen); From b5852dc6250278049cd94f3c91244a9be3e27015 Mon Sep 17 00:00:00 2001 From: Gabe Date: Fri, 25 Jul 2025 13:18:56 -0400 Subject: [PATCH 114/139] Fix formatting --- src/physics/degas2/DG2Physics.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 71374d7..733d2a8 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -238,9 +238,7 @@ class DG2Physics { //This definition may be redundant since it is also in the next location function particle_info.alpha = Kokkos::sqrt(mp/(2.0*particle_energy(particle_info.particle_index))); random_pool.free_state(rand_gen); - - -} + } //To here random_pool_t random_pool; DG2CrossSection cross_section; From 952647dbf79738694f190588adc5dbd5ff99f223 Mon Sep 17 00:00:00 2001 From: "Gabe J. Paynter" Date: Fri, 25 Jul 2025 18:16:48 -0400 Subject: [PATCH 115/139] Bug fix: random_pool, uniform source distribution, NEEDS TO FIX: illegal memory access --- src/physics/degas2/degas2-main.cpp | 12 +++-- src/pumitallyopenmc/pumitally_impl.cpp | 62 +++++++++++++++++++------- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 76f61b8..63581db 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -72,7 +72,7 @@ int main(int argc, char *argv[]) { // Initialize PUMI-Tally and Read Fields auto pumi_tally = pumiinopenmc::PumiTallyImpl( - input_params.mesh_name, input_params.num_particles, argc, argv); + input_params.mesh_name, input_params.num_particles, argc, argv, input_params.source_distribution); auto &mesh = pumi_tally.full_mesh_; Omega_h::Write centroids(mesh.nelems() * 3); get_centroids(mesh, centroids); @@ -135,11 +135,14 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, auto last_exit = pumi_tally.p_pumi_particle_at_elem_boundary_handler->last_exit_; + auto alpha = pumi_tally.p_pumi_particle_at_elem_boundary_handler->alpha_; auto get_new_destination = PS_LAMBDA(const int &e, const int &pid, const int &mask) { + if (mask > 0) { // FIXME: check if the particle is at destination or at // the boundary + // ParticleInfo particle_info; particle_info.position[0] = particle_dest(pid, 0); particle_info.position[1] = particle_dest(pid, 1); @@ -177,6 +180,7 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, particle_group(pid) = particle_info.energy_group; alpha[pid] = particle_info.alpha; + } }; pumipic::parallel_for(pumi_tally.pumipic_ptcls.get(), get_new_destination, @@ -411,8 +415,7 @@ void read_input_parameters(int argc, char *const *argv, } void sample_initial_particle_energy(Kokkos::View energy_array) { - random_pool_t randomPool; - + random_pool_t randomPool(0); auto sample_energy = OMEGA_H_LAMBDA(int i) { // Basically, feed this 4 uniformly generated random numbers, x1, x2, y1, // y2, on the interval (0,1) and it @@ -420,7 +423,6 @@ void sample_initial_particle_energy(Kokkos::View energy_array) { // (directionx, directiony, directionz), the alpha value used in the tally, // and the energy of the particle that was sampled. All from a gas at // temperature temp. - double mp{938.27e6 / (3e10 * 3e10)}; // eV/c^2 = eV*s^2/cm^2. Necessary // constant for the distribution @@ -465,6 +467,8 @@ void sample_initial_particle_energy(Kokkos::View energy_array) { // computations. double particle_energy = 0.5 * mp * mag_v * mag_v; energy_array(i) = particle_energy; + + }; Kokkos::parallel_for("sample_initial_particle_energy", energy_array.size(), sample_energy); diff --git a/src/pumitallyopenmc/pumitally_impl.cpp b/src/pumitallyopenmc/pumitally_impl.cpp index 9aae074..d49a3c4 100644 --- a/src/pumitallyopenmc/pumitally_impl.cpp +++ b/src/pumitallyopenmc/pumitally_impl.cpp @@ -592,18 +592,31 @@ void PumiParticleAtElemBoundary::updatePrevXPoint( void PumiParticleAtElemBoundary::updatePrevXPoint(PPPS *ptcls) { // todo add checks of size auto prev_xpoints_l = prev_xpoint_; + printf("\n Size of previous xpoints %d\n", prev_xpoints_l.size()); + printf("\nNumber of particles: %d\n", ptcls->nPtcls()); OMEGA_H_CHECK_PRINTF( ptcls->capacity() * 3 == prev_xpoints_l.size(), "Error: prev_xpoints_s are not size properly capacity %d size %d\n", ptcls->capacity(), prev_xpoints_l.size()); + printf("\n Omega_h Check Ran \n"); auto xpoints = ptcls->get<0>(); + printf("\n** xpoints defined **\n"); auto update = PS_LAMBDA(const auto &e, const auto &pid, const auto &mask) { + printf("\n** Inside lambda ** \n"); + if (mask>0 && pid<10) { + + printf("\n ***Copying xpoints***\n"); prev_xpoints_l[pid * 3 + 0] = xpoints(pid, 0); prev_xpoints_l[pid * 3 + 1] = xpoints(pid, 1); prev_xpoints_l[pid * 3 + 2] = xpoints(pid, 2); + } }; + printf("\n ** After the loop ** \n"); pumipic::parallel_for(ptcls, update, "update previous xpoints from origin points"); + printf("\nBefore fence \n"); + Kokkos::fence(); + printf("\n after fence \n"); } void PumiParticleAtElemBoundary::evaluateFlux( @@ -618,6 +631,7 @@ void PumiParticleAtElemBoundary::evaluateFlux( auto p_wgt = ptcls->get<4>(); auto p_groups = ptcls->get<5>(); auto xpoints_l = xpoints; // todo shouldn't need it, so remove + auto alpha_l = alpha_; auto evaluate_flux = PS_LAMBDA(const int &e, const int &pid, const int &mask) { @@ -648,7 +662,7 @@ void PumiParticleAtElemBoundary::evaluateFlux( orig[0], orig[1], orig[2], dest[0], dest[1], dest[2]); } - Omega_h::Real contribution = segment_length * p_wgt(pid) * alpha_[pid]; + Omega_h::Real contribution = segment_length * p_wgt(pid) * alpha_l[pid]; int group = p_groups(pid); OMEGA_H_CHECK_PRINTF( @@ -873,6 +887,7 @@ PumiPIC particle structure size to %d\n", n_particles); void start_pumi_particles_in_0th_element(Omega_h::Mesh &mesh, pumiinopenmc::PPPS *ptcls) { // find the centroid of the 0th element + printf("\n***Started 0th Element***\n"); const auto &coords = mesh.coords(); const auto &tet2node = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; @@ -943,12 +958,16 @@ void PumiTallyImpl::create_and_initialize_pumi_particle_structure( init_loc(pid, 0) = device_pos_buffer_l[pid * 3 + 0]; init_loc(pid, 1) = device_pos_buffer_l[pid * 3 + 1]; init_loc(pid, 2) = device_pos_buffer_l[pid * 3 + 2]; + if (pid < 10){ + printf("\n PID %d in element %d: (%f, %f, %f)\n",pid, e, init_loc(pid,0), init_loc(pid,1), init_loc(pid,2)); + } } }; pumipic::parallel_for(pumipic_ptcls.get(), copy_initial_positions, "copy initial positions from device buffer"); } if (source_dist == SourceDistribution::ZERO) { + printf("\**nSource Distribution 0**\n"); start_pumi_particles_in_0th_element(*mesh, pumipic_ptcls.get()); } p_pumi_particle_at_elem_boundary_handler = @@ -1090,6 +1109,7 @@ barycentric_basis(const o::Few, 4> &tet_verts) { return basis; } +/* OMEGA_H_DEVICE o::Vector<3> barycentric2real(const o::Few, 4> &tet_verts, const o::Vector<4> &bary) { @@ -1105,6 +1125,18 @@ barycentric2real(const o::Few, 4> &tet_verts, return real_coords; } +*/ + +OMEGA_H_DEVICE o::Vector<3> +barycentric2real(const o::Few, 4>& tet_verts, +const o::Vector<4>& bary) { +o::Vector<3> real_coords {0,0,0}; +for (int i = 0; i < 4; ++i) { +real_coords += bary[i] * tet_verts[i]; +} +return real_coords; +} + void initialize_uniform_source(Omega_h::Mesh &mesh, Omega_h::Write particle_positions, @@ -1135,7 +1167,7 @@ void initialize_uniform_source(Omega_h::Mesh &mesh, const auto cells2nodes = mesh.ask_down(o::REGION, o::VERT).ab2b; const auto coords = mesh.coords(); - Kokkos::Random_XorShift64_Pool random_pool; + Kokkos::Random_XorShift64_Pool random_pool(0); auto set_initial_positions = OMEGA_H_LAMBDA(const int &e) { auto pid_start = cumulative_particles[e]; auto pid_end = cumulative_particles[e + 1]; @@ -1144,19 +1176,19 @@ void initialize_uniform_source(Omega_h::Mesh &mesh, for (Omega_h::LO pid = pid_start; pid < pid_end; ++pid) { auto gen = random_pool.get_state(); - o::Vector<4> random_bcc{0.0, 0.0, 0.0, 0.0}; - random_bcc[0] = gen.drand(0.0, 1.0); - random_bcc[1] = gen.drand(0.0, 1.0); - random_bcc[2] = gen.drand(0.0, 1.0); - o::Real complimentary0 = 1.0 - random_bcc[0]; - o::Real complimentary1 = 1.0 - random_bcc[1]; - o::Real complimentary2 = 1.0 - random_bcc[2]; - - bool more_than_one = random_bcc[0] + random_bcc[1] + random_bcc[2] > 1.0; - random_bcc[0] = more_than_one ? complimentary0 : random_bcc[0]; - random_bcc[1] = more_than_one ? complimentary1 : random_bcc[1]; - random_bcc[2] = more_than_one ? complimentary2 : random_bcc[2]; - random_bcc[2] = 1.0 - random_bcc[0] - random_bcc[1] - random_bcc[2]; + o::Real r1 = gen.drand(0.0, 1.0); + o::Real r2 = gen.drand(0.0, 1.0); + o::Real r3 = gen.drand(0.0, 1.0); + + r1 = Kokkos::pow(r1, 1.0 / 3.0); + r2 = Kokkos::sqrt(r2); + o::Real a = 1.0 - r1; + o::Real b = r1 * (1.0 - r2); + o::Real c = r1 * r2 * (1.0 - r3); + o::Real d = r1 * r2 * r3; + + o::Vector<4> random_bcc{a, b, c, d}; + random_pool.free_state(gen); auto verts = o::gather_verts<4>(cells2nodes, e); From 6c09ac0d6e53f890eb97ee8bb6fb98ea1ef2f2a2 Mon Sep 17 00:00:00 2001 From: Gabe Paynter Date: Tue, 29 Jul 2025 20:32:58 -0400 Subject: [PATCH 116/139] Fix illegal memory access error. NEED TO FIX: particles deleting --- src/physics/degas2/DG2Physics.h | 12 ++++++------ src/physics/degas2/degas2-main.cpp | 18 +++++++++++------- src/pumitallyopenmc/pumitally_impl.cpp | 20 +++++--------------- 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 71374d7..29d5116 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -151,7 +151,7 @@ class DG2Physics { double sigma_cx = charge_exchange_cross_section(particle_info, field_info); //Generate distance and move particle - double l =-Kokkos::log(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx); //cm. n in cm^-3 + double l =-Kokkos::log(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx)*0.01; //m. n in cm^-3 particle_info.position[0] += l*particle_info.direction[0]; particle_info.position[1] += l*particle_info.direction[1]; particle_info.position[2] += l*particle_info.direction[2]; @@ -176,12 +176,12 @@ class DG2Physics { //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 double max_sigma_cx = 3.8e-14; - + bool rejection_test = false; auto old_mag_v = Kokkos::sqrt(2*particle_energy(particle_info.particle_index)/mp); - double vx; - double vy; - double vz; + double vx {}; + double vy {}; + double vz {}; //Loops this until it passes the rejection test while (!rejection_test) { //Generate random numbers for the velocity @@ -213,7 +213,7 @@ class DG2Physics { particle_info.direction[0] = vx/mag_v; particle_info.direction[1] = vy/mag_v; particle_info.direction[2] = vz/mag_v; - + particle_energy(particle_info.particle_index) = 0.5*mp*mag_v*mag_v; //particle_info.energy_group = particle_energy(particle_info.particle_index); //Temporary for debugging //Adjust Weights diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 63581db..2def0be 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -166,12 +166,17 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, field_info.bulk_flow_velocity[0] = bulk_flow_velocity[e * 3 + 0]; field_info.bulk_flow_velocity[1] = bulk_flow_velocity[e * 3 + 1]; field_info.bulk_flow_velocity[2] = bulk_flow_velocity[e * 3 + 2]; - - if (last_exit[pid] == -1) { // reached destination - physics.collide_particle(particle_info, field_info); - } + + if (iter != 0) { //Ensure last_exit[pid] doesn't run the first time + if (last_exit[pid] == -1) { // reached destination + physics.collide_particle(particle_info, field_info); + } + } + else { + physics.collide_particle(particle_info, field_info); + } physics.sample_collision_distance(particle_info, field_info); - + // Update particle position and direction particle_dest(pid, 0) = particle_info.position[0]; particle_dest(pid, 1) = particle_info.position[1]; @@ -180,12 +185,11 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, particle_group(pid) = particle_info.energy_group; alpha[pid] = particle_info.alpha; - + printf("\n#### PID: %d, iter: %d, at position (%f,%f,%f) ####\n", pid, iter, particle_info.position[0], particle_info.position[1], particle_info.position[2]); } }; pumipic::parallel_for(pumi_tally.pumipic_ptcls.get(), get_new_destination, "get new destination"); - pumi_tally.search_and_rebuild( false, true); // for now, always rebuild the pp structure Kokkos::fence(); diff --git a/src/pumitallyopenmc/pumitally_impl.cpp b/src/pumitallyopenmc/pumitally_impl.cpp index d49a3c4..42430d0 100644 --- a/src/pumitallyopenmc/pumitally_impl.cpp +++ b/src/pumitallyopenmc/pumitally_impl.cpp @@ -592,31 +592,21 @@ void PumiParticleAtElemBoundary::updatePrevXPoint( void PumiParticleAtElemBoundary::updatePrevXPoint(PPPS *ptcls) { // todo add checks of size auto prev_xpoints_l = prev_xpoint_; - printf("\n Size of previous xpoints %d\n", prev_xpoints_l.size()); - printf("\nNumber of particles: %d\n", ptcls->nPtcls()); OMEGA_H_CHECK_PRINTF( ptcls->capacity() * 3 == prev_xpoints_l.size(), "Error: prev_xpoints_s are not size properly capacity %d size %d\n", ptcls->capacity(), prev_xpoints_l.size()); - printf("\n Omega_h Check Ran \n"); auto xpoints = ptcls->get<0>(); - printf("\n** xpoints defined **\n"); auto update = PS_LAMBDA(const auto &e, const auto &pid, const auto &mask) { - printf("\n** Inside lambda ** \n"); if (mask>0 && pid<10) { - printf("\n ***Copying xpoints***\n"); prev_xpoints_l[pid * 3 + 0] = xpoints(pid, 0); prev_xpoints_l[pid * 3 + 1] = xpoints(pid, 1); prev_xpoints_l[pid * 3 + 2] = xpoints(pid, 2); } }; - printf("\n ** After the loop ** \n"); pumipic::parallel_for(ptcls, update, - "update previous xpoints from origin points"); - printf("\nBefore fence \n"); - Kokkos::fence(); - printf("\n after fence \n"); + "update previous xpoints from origin points"); } void PumiParticleAtElemBoundary::evaluateFlux( @@ -958,16 +948,16 @@ void PumiTallyImpl::create_and_initialize_pumi_particle_structure( init_loc(pid, 0) = device_pos_buffer_l[pid * 3 + 0]; init_loc(pid, 1) = device_pos_buffer_l[pid * 3 + 1]; init_loc(pid, 2) = device_pos_buffer_l[pid * 3 + 2]; - if (pid < 10){ - printf("\n PID %d in element %d: (%f, %f, %f)\n",pid, e, init_loc(pid,0), init_loc(pid,1), init_loc(pid,2)); - } +// if (pid < 10){ +// printf("\n PID %d in element %d: (%f, %f, %f)\n",pid, e, init_loc(pid,0), init_loc(pid,1), init_loc(pid,2)); +// } } }; pumipic::parallel_for(pumipic_ptcls.get(), copy_initial_positions, "copy initial positions from device buffer"); } if (source_dist == SourceDistribution::ZERO) { - printf("\**nSource Distribution 0**\n"); + printf("\n**Source Distribution 0**\n"); start_pumi_particles_in_0th_element(*mesh, pumipic_ptcls.get()); } p_pumi_particle_at_elem_boundary_handler = From f13992a890dc0d902471a4f6ca3b4bc7f74829ec Mon Sep 17 00:00:00 2001 From: Gabe Paynter Date: Fri, 1 Aug 2025 14:29:08 -0400 Subject: [PATCH 117/139] Fix particle deleting due to improper definitions of igin and destination position. NEED TO FIX: new issue of particles deleting --- src/physics/degas2/DG2Physics.h | 9 ++++++--- src/physics/degas2/degas2-main.cpp | 23 +++++++++++++++-------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index a1c160c..2e5f2d9 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -151,13 +151,16 @@ class DG2Physics { double sigma_cx = charge_exchange_cross_section(particle_info, field_info); //Generate distance and move particle +// printf("\n$$$ Before moving particle [%d], position: (%f, %f, %f) $$$\n", particle_info.particle_index, particle_info.position[0], particle_info.position[1], +// particle_info.position[2]); double l =-Kokkos::log(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx)*0.01; //m. n in cm^-3 particle_info.position[0] += l*particle_info.direction[0]; particle_info.position[1] += l*particle_info.direction[1]; particle_info.position[2] += l*particle_info.direction[2]; - +// printf("\n$$$ After moving particle [%d], position (%f, %f, %f). Direction was (%f, %f, %f) with l = %f $$$\n", particle_info.particle_index, particle_info.position[0], +// particle_info.position[1], particle_info.position[2], particle_info.direction[0], particle_info.direction[1], particle_info.direction[2], l); double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - particle_info.alpha = Kokkos::sqrt(mp/(2.0*particle_energy(particle_info.particle_index))); + particle_info.alpha = Kokkos::sqrt(mp/(2.0*particle_energy(particle_info.particle_index)))*100; //1/(m/s) } // collision event @@ -236,7 +239,7 @@ class DG2Physics { particle_info.weight = new_weight; //This definition may be redundant since it is also in the next location function - particle_info.alpha = Kokkos::sqrt(mp/(2.0*particle_energy(particle_info.particle_index))); + particle_info.alpha = Kokkos::sqrt(mp/(2.0*particle_energy(particle_info.particle_index)))*100; //1/(m/s) random_pool.free_state(rand_gen); } //To here diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 2def0be..755b15e 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -136,21 +136,23 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, auto last_exit = pumi_tally.p_pumi_particle_at_elem_boundary_handler->last_exit_; + printf("\n### iter = %d ###\n",iter); + auto alpha = pumi_tally.p_pumi_particle_at_elem_boundary_handler->alpha_; auto get_new_destination = PS_LAMBDA(const int &e, const int &pid, const int &mask) { - + if (mask > 0) { // FIXME: check if the particle is at destination or at // the boundary // ParticleInfo particle_info; - particle_info.position[0] = particle_dest(pid, 0); - particle_info.position[1] = particle_dest(pid, 1); - particle_info.position[2] = particle_dest(pid, 2); + particle_info.position[0] = particle_orig(pid, 0); + particle_info.position[1] = particle_orig(pid, 1); + particle_info.position[2] = particle_orig(pid, 2); auto direction = Omega_h::normalize(Omega_h::Vector<3>{ - particle_info.position[0] - particle_orig(pid, 0), - particle_info.position[1] - particle_orig(pid, 1), - particle_info.position[2] - particle_orig(pid, 2)}); + particle_info.position[0] - particle_dest(pid, 0), + particle_info.position[1] - particle_dest(pid, 1), + particle_info.position[2] - particle_dest(pid, 2)}); particle_info.direction[0] = direction[0]; particle_info.direction[1] = direction[1]; particle_info.direction[2] = direction[2]; @@ -167,6 +169,10 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, field_info.bulk_flow_velocity[1] = bulk_flow_velocity[e * 3 + 1]; field_info.bulk_flow_velocity[2] = bulk_flow_velocity[e * 3 + 2]; + +// printf("\n---- Direction (pid = %d) before collision: (%f, %f, %f). Original pos: (%f, %f, %f), pos (%f, %f, %f) ---- \n", pid, direction[0], direction[1], direction[2], +// particle_orig(pid,0), particle_orig(pid,1), particle_orig(pid,2), particle_info.position[0], particle_info.position[1], particle_info.position[2]); + if (iter != 0) { //Ensure last_exit[pid] doesn't run the first time if (last_exit[pid] == -1) { // reached destination physics.collide_particle(particle_info, field_info); @@ -185,7 +191,8 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, particle_group(pid) = particle_info.energy_group; alpha[pid] = particle_info.alpha; - printf("\n#### PID: %d, iter: %d, at position (%f,%f,%f) ####\n", pid, iter, particle_info.position[0], particle_info.position[1], particle_info.position[2]); +// printf("\n#### PID: %d, iter: %d, at position (%f,%f,%f). Direction was: (%f, %f, %f) ####\n", pid, iter, particle_info.position[0], +// particle_info.position[1], particle_info.position[2], particle_info.direction[0], particle_info.direction[1], particle_info.direction[2]); } }; pumipic::parallel_for(pumi_tally.pumipic_ptcls.get(), get_new_destination, From e66f8a862622a6d25ac0abccfdb7a3a59247e055 Mon Sep 17 00:00:00 2001 From: gpaynter5004 Date: Fri, 8 Aug 2025 11:54:37 -0400 Subject: [PATCH 118/139] Re-enable the print for the test test_degas2_physics.cpp --- test/test_degas2_physics.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 28154e1..03d853d 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -61,7 +61,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ }); auto output = create_mirror_view(particles); Kokkos::deep_copy(output, particles); - //std::ofstream outfile("Log.txt"); + std::ofstream outfile("Log.txt"); //These 4 functions should return the commented value for SEED=12345 @@ -70,7 +70,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ l += output(i).position[0]; } l /= numParticles; - //outfile << "Average Distance (cm): " << l << std::endl; //2.41 + outfile << "Average Distance (cm): " << l << std::endl; //2.41 double varl {0}; for (int i=0; i < numParticles; ++i) { @@ -78,14 +78,14 @@ TEST_CASE("Test Degas2 Physics Functions"){ } varl /= (numParticles - 1); double sdl {sqrt(varl)}; - //outfile << "Standard Deviation of Distance (cm): " << sdl << std::endl; //2.38 + outfile << "Standard Deviation of Distance (cm): " << sdl << std::endl; //2.38 double ux {0}; for (int i=0; i < numParticles; ++i) { ux += output(i).direction[0]; } ux /= numParticles; - //outfile << "Mean x Direction: " << ux << std::endl; //0.0159 + outfile << "Mean x Direction: " << ux << std::endl; //0.0159 double varux; for (int i=0; i < numParticles; ++i) { @@ -94,7 +94,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ varux /= (numParticles - 1); double sdux {sqrt(varux)}; - /* + outfile << "Standard Deviation of Mean x Direction: " << sdux << std::endl; //0.579 @@ -102,7 +102,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ for (int i = 0; i < numParticles; ++i) { outfile << i << ","; - outfile << output(i).energy_group << ","; + outfile << (1.0/output(i).alpha) << ","; outfile << output(i).weight << ","; outfile << output(i).position[0] << ","; outfile << output(i).position[1] << ","; @@ -111,7 +111,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ outfile << output(i).direction[1] << ","; outfile << output(i).direction[2] << std::endl; } - */ + REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.04)); REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.04)); REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); From 2ad7a739874a54abf2abde69d9bec168df5be358 Mon Sep 17 00:00:00 2001 From: gpaynter5004 Date: Fri, 8 Aug 2025 11:57:58 -0400 Subject: [PATCH 119/139] Correct rejection sampling DG2Physics.h --- src/physics/degas2/DG2Physics.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 2e5f2d9..58b8d05 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -178,7 +178,7 @@ class DG2Physics { //Compute New Direction and Energy and set particle info //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double max_sigma_cx = 3.8e-14; + double max_rate_cx = 1.4e-7; bool rejection_test = false; auto old_mag_v = Kokkos::sqrt(2*particle_energy(particle_info.particle_index)/mp); @@ -206,7 +206,7 @@ class DG2Physics { auto mag_v2 = rel_vx*rel_vx + rel_vy*rel_vy + rel_vz*rel_vz; //Generate random number and compare to sigma/sigma_max - if (rand_gen.drand(0., 1.) < analytic_cross_section_CE(0.5*mp*mag_v2)/max_sigma_cx) { + if (rand_gen.drand(0., 1.) < mag_v2*analytic_cross_section_CE(0.5*mp*mag_v2)/max_rate_cx) { rejection_test = true; } } From 29fa14efc7f615be3be700db65b56516f68ebb2d Mon Sep 17 00:00:00 2001 From: gpaynter5004 Date: Fri, 8 Aug 2025 12:17:59 -0400 Subject: [PATCH 120/139] Correct units in test case test_degas2_physics.cpp --- test/test_degas2_physics.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 03d853d..9b107b2 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -112,8 +112,8 @@ TEST_CASE("Test Degas2 Physics Functions"){ outfile << output(i).direction[2] << std::endl; } - REQUIRE_THAT(l, Catch::Matchers::WithinAbs(2.40,.04)); - REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(2.40,.04)); + REQUIRE_THAT(l, Catch::Matchers::WithinAbs(0.0240,0.0004)); + REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(0.0240,0.0004)); REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); } From 7c601900bb8f83649e386c91a3f8d196e2c6ebbe Mon Sep 17 00:00:00 2001 From: gpaynter5004 Date: Fri, 8 Aug 2025 12:21:04 -0400 Subject: [PATCH 121/139] Correct units in log test_degas2_physics.cpp --- test/test_degas2_physics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 9b107b2..daef4d7 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -98,7 +98,7 @@ TEST_CASE("Test Degas2 Physics Functions"){ outfile << "Standard Deviation of Mean x Direction: " << sdux << std::endl; //0.579 - outfile << "Particle #,Energy(eV),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; + outfile << "Particle #,Velocity(m/s),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; for (int i = 0; i < numParticles; ++i) { outfile << i << ","; From b401232bf274730549ae78dbb326cb87768ee407 Mon Sep 17 00:00:00 2001 From: gpaynter5004 Date: Fri, 8 Aug 2025 12:46:53 -0400 Subject: [PATCH 122/139] Bug fix rejection sampling DG2Physics.h --- src/physics/degas2/DG2Physics.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index 58b8d05..e1549ae 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -206,7 +206,7 @@ class DG2Physics { auto mag_v2 = rel_vx*rel_vx + rel_vy*rel_vy + rel_vz*rel_vz; //Generate random number and compare to sigma/sigma_max - if (rand_gen.drand(0., 1.) < mag_v2*analytic_cross_section_CE(0.5*mp*mag_v2)/max_rate_cx) { + if (rand_gen.drand(0., 1.) < Kokkos::sqrt(mag_v2)*analytic_cross_section_CE(0.5*mp*mag_v2)/max_rate_cx) { rejection_test = true; } } From 9ee65a660fa5f1a3f5fa0f4331bde2c8c0158f4c Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Wed, 20 Aug 2025 21:06:30 -0400 Subject: [PATCH 123/139] fix bugs - initialization fixes for particles - fix reflection boundary condition --- src/physics/degas2/degas2-main.cpp | 12 +++++------ src/pumitallyopenmc/pumitally_impl.cpp | 30 ++++++++++++++++++++------ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 755b15e..40ff987 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -177,10 +177,10 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, if (last_exit[pid] == -1) { // reached destination physics.collide_particle(particle_info, field_info); } - } - else { - physics.collide_particle(particle_info, field_info); - } + } else { + physics.collide_particle(particle_info, field_info); + } + physics.sample_collision_distance(particle_info, field_info); // Update particle position and direction @@ -188,11 +188,9 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, particle_dest(pid, 1) = particle_info.position[1]; particle_dest(pid, 2) = particle_info.position[2]; particle_weight(pid) = particle_info.weight; - particle_group(pid) = particle_info.energy_group; + particle_group(pid) = 0; // physics kernel does not handle groups yet alpha[pid] = particle_info.alpha; -// printf("\n#### PID: %d, iter: %d, at position (%f,%f,%f). Direction was: (%f, %f, %f) ####\n", pid, iter, particle_info.position[0], -// particle_info.position[1], particle_info.position[2], particle_info.direction[0], particle_info.direction[1], particle_info.direction[2]); } }; pumipic::parallel_for(pumi_tally.pumipic_ptcls.get(), get_new_destination, diff --git a/src/pumitallyopenmc/pumitally_impl.cpp b/src/pumitallyopenmc/pumitally_impl.cpp index 42430d0..f8fa172 100644 --- a/src/pumitallyopenmc/pumitally_impl.cpp +++ b/src/pumitallyopenmc/pumitally_impl.cpp @@ -494,16 +494,23 @@ void apply_reflection_boundary_condition( // Material (%3d -> %3d)\n", // pid, elem_ids[pid], next_elems[pid], // class_ids[elem_ids[pid]], material_ids[pid]); - xFace[pid] = lastExit[pid]; - OMEGA_H_CHECK_PRINTF(lastExit[pid] != -1, - "Error: lastExit[%d] is -1 but " + //printf("P %d in element %d hit lastExit %d xFace %d \n", pid, + // elem_ids[pid], lastExit[pid], xFace[pid]); + //xFace[pid] = lastExit[pid]; + Omega_h::LO hit_face = (lastExit[pid] == -1) ? xFace[pid] : lastExit[pid]; + xFace[pid] = hit_face; + lastExit[pid] = hit_face; + ptcl_done[pid] = 1; // stop the particle after reflection + next_elems[pid] = elem_ids[pid]; // reflects back to the same element + OMEGA_H_CHECK_PRINTF(hit_face != -1, + "Error: xFace[%d] is -1 but " "hit_outer_boundary is true\n", pid); // change direction - auto normal = Omega_h::Vector<3>{normals[lastExit[pid] * 3 + 0], - normals[lastExit[pid] * 3 + 1], - normals[lastExit[pid] * 3 + 2]}; + auto normal = Omega_h::Vector<3>{normals[hit_face * 3 + 0], + normals[hit_face * 3 + 1], + normals[hit_face * 3 + 2]}; Omega_h::Vector<3> particle_direction = { particle_destination(pid, 0) - particle_origin(pid, 0), particle_destination(pid, 1) - particle_origin(pid, 1), @@ -639,6 +646,7 @@ void PumiParticleAtElemBoundary::evaluateFlux( Omega_h::Real segment_length = Omega_h::norm(dest - orig); // / total_particles; + /* if (segment_length > total_tracklength_l[pid] + 1e-6) { // tol for float operations and // search algorithm's inaccuracy @@ -651,6 +659,7 @@ void PumiParticleAtElemBoundary::evaluateFlux( segment_length, total_tracklength_l[pid], pid, elem_ids[pid], e, orig[0], orig[1], orig[2], dest[0], dest[1], dest[2]); } + */ Omega_h::Real contribution = segment_length * p_wgt(pid) * alpha_l[pid]; @@ -715,7 +724,7 @@ void PumiParticleAtElemBoundary::finalizeAndWritePumiFlux( Omega_h::Write normalized_flux(num_elems, 0.0, "normalized_flux"); // copy flux to normalized_flux - for (int g = 0; g < num_groups; g++) { + for (int g = 0; g < 1; g++) { auto copy_flux = OMEGA_H_LAMBDA(Omega_h::LO elem_id) { normalized_flux[elem_id] = flux(elem_id, g, 0); }; @@ -724,8 +733,10 @@ void PumiParticleAtElemBoundary::finalizeAndWritePumiFlux( Kokkos::fence(); full_mesh.add_tag(Omega_h::REGION, tag_name, 1, Omega_h::Reals(normalized_flux)); + Kokkos::fence(); } Omega_h::vtk::write_parallel(filename, &full_mesh, 3); + Kokkos::fence(); } void pumiUpdatePtclPositions(PPPS *ptcls) { @@ -940,6 +951,8 @@ void PumiTallyImpl::create_and_initialize_pumi_particle_structure( // copy the device positions to the particle structure auto init_loc = pumipic_ptcls->get<0>(); auto pids = pumipic_ptcls->get<2>(); + auto in_flight = pumipic_ptcls->get<3>(); + auto weight = pumipic_ptcls->get<4>(); auto copy_initial_positions = PS_LAMBDA(const int &e, const int &pid, const int &mask) { @@ -948,6 +961,9 @@ void PumiTallyImpl::create_and_initialize_pumi_particle_structure( init_loc(pid, 0) = device_pos_buffer_l[pid * 3 + 0]; init_loc(pid, 1) = device_pos_buffer_l[pid * 3 + 1]; init_loc(pid, 2) = device_pos_buffer_l[pid * 3 + 2]; + + in_flight(pid) = 1; // initialize the particle as in flight + weight(pid) = 1.0; // initialize the particle weight to 1.0 // if (pid < 10){ // printf("\n PID %d in element %d: (%f, %f, %f)\n",pid, e, init_loc(pid,0), init_loc(pid,1), init_loc(pid,2)); // } From 160789099741a42daafa1f124127dc22bb5b25bf Mon Sep 17 00:00:00 2001 From: Fuad Hasibul Hasan Date: Fri, 22 Aug 2025 15:35:18 -0400 Subject: [PATCH 124/139] use isotropically sampled initial direction -fix kokkos finalization --- src/physics/degas2/degas2-main.cpp | 161 +++++++++++++------------ src/pumitallyopenmc/pumitally_impl.cpp | 14 +-- 2 files changed, 94 insertions(+), 81 deletions(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 40ff987..6242a37 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -29,7 +29,9 @@ void print_initial_info(const std::string &mesh_name, // computes centroids of each element in the mesh and stores them in a tag void get_centroids(Omega_h::Mesh &mesh, Omega_h::Write centroids); -void sample_initial_particle_energy(Kokkos::View energy_array); +void +sample_initial_particle_energy_direction(Kokkos::View energy_array, Kokkos::View direction, + Kokkos::View alpha); template void vector2write(const std::vector &vec, Omega_h::Write &write_vec) { @@ -52,62 +54,67 @@ void set_field_values_to_mesh(Omega_h::Mesh &mesh, const Fields &fields); // void set_source_particles(Omega_h::Mesh &mesh, // SourceDistribution source_distribution, // Omega_h::Write particle_positions); -void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, - const Omega_h::Read &electron_density, - const Omega_h::Read ion_density, - const Omega_h::Read electron_temperature, - const Omega_h::Read ion_temperature, - const Omega_h::Read bulk_flow_velocity, - int max_iterations); +void +transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, const Omega_h::Read &electron_density, + const Omega_h::Read ion_density, const Omega_h::Read electron_temperature, + const Omega_h::Read ion_temperature, const Omega_h::Read bulk_flow_velocity, + const Kokkos::View initial_direction, const Kokkos::View initial_alpha, + int max_iterations); // ***************************************************************************// // *************************** Main Function ******************************** // // ***************************************************************************// int main(int argc, char *argv[]) { - - // Read input parameters - InputParameters input_params; - read_input_parameters(argc, argv, input_params); - print_initial_info(input_params.mesh_name, input_params.num_particles); - - // Initialize PUMI-Tally and Read Fields - auto pumi_tally = pumiinopenmc::PumiTallyImpl( - input_params.mesh_name, input_params.num_particles, argc, argv, input_params.source_distribution); - auto &mesh = pumi_tally.full_mesh_; - Omega_h::Write centroids(mesh.nelems() * 3); - get_centroids(mesh, centroids); - Fields fields; - get_field_values(Omega_h::Reals(centroids), fields); - set_field_values_to_mesh(mesh, fields); - pumi_tally.is_pumipic_initialized = true; - - Kokkos::View sigma_t_; // mat, T, g - Kokkos::View sigma_a_; // mat, T, g - Kokkos::View scattering_matrix_; // mat, T, g, g - Kokkos::View sigma_s_; // mat, T, g - DG2CrossSection crossSection(sigma_t_, sigma_a_, scattering_matrix_, - sigma_s_); - DG2Physics physics(crossSection, input_params.num_particles); - sample_initial_particle_energy(physics.particle_energy); - - // Move particles - auto electron_density = - mesh.get_array(Omega_h::REGION, "electron_density"); - auto ion_density = - mesh.get_array(Omega_h::REGION, "ion_density"); - auto electron_temperature = - mesh.get_array(Omega_h::REGION, "electron_temperature"); - auto ion_temperature = - mesh.get_array(Omega_h::REGION, "ion_temperature"); - auto bulk_flow_velocity = - mesh.get_array(Omega_h::REGION, "bulk_flow_velocity"); - - transport(pumi_tally, physics, electron_density, ion_density, - electron_temperature, ion_temperature, bulk_flow_velocity, - input_params.max_iterations); - - // Finalize and output results - pumi_tally.write_pumi_tally_mesh(); + Kokkos::initialize(argc, argv); + { + // Read input parameters + InputParameters input_params; + read_input_parameters(argc, argv, input_params); + print_initial_info(input_params.mesh_name, input_params.num_particles); + + // Initialize PUMI-Tally and Read Fields + auto pumi_tally = pumiinopenmc::PumiTallyImpl( + input_params.mesh_name, input_params.num_particles, argc, argv, input_params.source_distribution); + auto &mesh = pumi_tally.full_mesh_; + Omega_h::Write centroids(mesh.nelems() * 3); + get_centroids(mesh, centroids); + Fields fields; + get_field_values(Omega_h::Reals(centroids), fields); + set_field_values_to_mesh(mesh, fields); + pumi_tally.is_pumipic_initialized = true; + + Kokkos::View < Omega_h::Real * ** > sigma_t_; // mat, T, g + Kokkos::View < Omega_h::Real * ** > sigma_a_; // mat, T, g + Kokkos::View < Omega_h::Real * *** > scattering_matrix_; // mat, T, g, g + Kokkos::View < Omega_h::Real * ** > sigma_s_; // mat, T, g + DG2CrossSection crossSection(sigma_t_, sigma_a_, scattering_matrix_, + sigma_s_); + DG2Physics physics(crossSection, input_params.num_particles); + + Kokkos::View initial_direction("Initial Direction", input_params.num_particles * 3); + Kokkos::View initial_alpha("Initial Alpha", input_params.num_particles); + sample_initial_particle_energy_direction(physics.particle_energy, initial_direction, initial_alpha); + + // Move particles + auto electron_density = + mesh.get_array(Omega_h::REGION, "electron_density"); + auto ion_density = + mesh.get_array(Omega_h::REGION, "ion_density"); + auto electron_temperature = + mesh.get_array(Omega_h::REGION, "electron_temperature"); + auto ion_temperature = + mesh.get_array(Omega_h::REGION, "ion_temperature"); + auto bulk_flow_velocity = + mesh.get_array(Omega_h::REGION, "bulk_flow_velocity"); + + transport(pumi_tally, physics, electron_density, ion_density, + electron_temperature, ion_temperature, bulk_flow_velocity, initial_direction, + initial_alpha, input_params.max_iterations); + + // Finalize and output results + pumi_tally.write_pumi_tally_mesh(); + } + Kokkos::finalize(); return 0; } @@ -119,13 +126,12 @@ int main(int argc, char *argv[]) { // **************************************************************************************************************** // // -void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, - const Omega_h::Read &electron_density, - const Omega_h::Read ion_density, - const Omega_h::Read electron_temperature, - const Omega_h::Read ion_temperature, - const Omega_h::Read bulk_flow_velocity, - int max_iterations) { +void +transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, const Omega_h::Read &electron_density, + const Omega_h::Read ion_density, const Omega_h::Read electron_temperature, + const Omega_h::Read ion_temperature, const Omega_h::Read bulk_flow_velocity, + const Kokkos::View initial_direction, const Kokkos::View initial_alpha, + int max_iterations) { for (int iter = 0; iter < max_iterations; ++iter) { auto particle_dest = pumi_tally.pumipic_ptcls->get<1>(); @@ -149,13 +155,20 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, particle_info.position[0] = particle_orig(pid, 0); particle_info.position[1] = particle_orig(pid, 1); particle_info.position[2] = particle_orig(pid, 2); - auto direction = Omega_h::normalize(Omega_h::Vector<3>{ + + if (iter == 0) { + particle_info.direction[0] = initial_direction(pid * 3 + 0); + particle_info.direction[1] = initial_direction(pid * 3 + 1); + particle_info.direction[2] = initial_direction(pid * 3 + 2); + } else { + auto direction = Omega_h::normalize(Omega_h::Vector < 3 > { particle_info.position[0] - particle_dest(pid, 0), particle_info.position[1] - particle_dest(pid, 1), particle_info.position[2] - particle_dest(pid, 2)}); - particle_info.direction[0] = direction[0]; - particle_info.direction[1] = direction[1]; - particle_info.direction[2] = direction[2]; + particle_info.direction[0] = direction[0]; + particle_info.direction[1] = direction[1]; + particle_info.direction[2] = direction[2]; + } particle_info.weight = particle_weight(pid); particle_info.energy_group = particle_group(pid); particle_info.particle_index = pid; @@ -173,12 +186,10 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, // printf("\n---- Direction (pid = %d) before collision: (%f, %f, %f). Original pos: (%f, %f, %f), pos (%f, %f, %f) ---- \n", pid, direction[0], direction[1], direction[2], // particle_orig(pid,0), particle_orig(pid,1), particle_orig(pid,2), particle_info.position[0], particle_info.position[1], particle_info.position[2]); - if (iter != 0) { //Ensure last_exit[pid] doesn't run the first time + if (iter != 0) { // first iteration, we do not need to collide if (last_exit[pid] == -1) { // reached destination physics.collide_particle(particle_info, field_info); } - } else { - physics.collide_particle(particle_info, field_info); } physics.sample_collision_distance(particle_info, field_info); @@ -190,7 +201,7 @@ void transport(pumiinopenmc::PumiTallyImpl &pumi_tally, DG2Physics &physics, particle_weight(pid) = particle_info.weight; particle_group(pid) = 0; // physics kernel does not handle groups yet - alpha[pid] = particle_info.alpha; + alpha[pid] = (iter == 0) ? initial_alpha(pid) : particle_info.alpha; } }; pumipic::parallel_for(pumi_tally.pumipic_ptcls.get(), get_new_destination, @@ -361,7 +372,7 @@ void get_field_values(Omega_h::Reals centroids, Fields &fields) { coords.close(); // Run the python script - system("python Mesh_Map.py"); + // system("python Mesh_Map.py"); // Read in the field values std::ifstream fieldvals{"vals.csv"}; @@ -423,7 +434,9 @@ void read_input_parameters(int argc, char *const *argv, } } -void sample_initial_particle_energy(Kokkos::View energy_array) { +void +sample_initial_particle_energy_direction(Kokkos::View energy_array, Kokkos::View direction, + Kokkos::View alpha) { random_pool_t randomPool(0); auto sample_energy = OMEGA_H_LAMBDA(int i) { // Basically, feed this 4 uniformly generated random numbers, x1, x2, y1, @@ -464,12 +477,12 @@ void sample_initial_particle_energy(Kokkos::View energy_array) { auto mag_v = Kokkos::sqrt(vx * vx + vy * vy + vz * vz); // Compute the alpha factor multiplied in when computing the tally - double alpha = 1 / mag_v; + alpha(i) = 1 / mag_v; // Compute the unit normal - double directionx = vx / mag_v; - double directiony = vy / mag_v; - double directionz = vz / mag_v; + direction(i * 3 + 0) = vx / mag_v; + direction(i * 3 + 1) = vy / mag_v; + direction(i * 3 + 2) = vz / mag_v; // Compute the particle energy. This you will want to save probably to pass // to my initialization (set_energy) code so that I have the energy for my @@ -479,6 +492,6 @@ void sample_initial_particle_energy(Kokkos::View energy_array) { }; - Kokkos::parallel_for("sample_initial_particle_energy", energy_array.size(), + Kokkos::parallel_for("sample_initial_particle_energy_direction", energy_array.size(), sample_energy); } diff --git a/src/pumitallyopenmc/pumitally_impl.cpp b/src/pumitallyopenmc/pumitally_impl.cpp index f8fa172..2ef53fb 100644 --- a/src/pumitallyopenmc/pumitally_impl.cpp +++ b/src/pumitallyopenmc/pumitally_impl.cpp @@ -511,14 +511,14 @@ void apply_reflection_boundary_condition( auto normal = Omega_h::Vector<3>{normals[hit_face * 3 + 0], normals[hit_face * 3 + 1], normals[hit_face * 3 + 2]}; - Omega_h::Vector<3> particle_direction = { + Omega_h::Vector<3> incident_vector = { particle_destination(pid, 0) - particle_origin(pid, 0), particle_destination(pid, 1) - particle_origin(pid, 1), particle_destination(pid, 2) - particle_origin(pid, 2)}; // reflect the particle direction - Omega_h::Vector<3> reflected_direction = - particle_direction - - 2.0 * Omega_h::inner_product(particle_direction, normal) * normal; + Omega_h::Vector<3> reflected_vector = + incident_vector - + 2.0 * Omega_h::inner_product(incident_vector, normal) * normal; // change the particle's position // particle reaches the boundary @@ -527,11 +527,11 @@ void apply_reflection_boundary_condition( particle_origin(pid, 2) = inter_points[pid * 3 + 2]; particle_destination(pid, 0) = - particle_origin(pid, 0) + reflected_direction[0]; + particle_origin(pid, 0) + reflected_vector[0]; particle_destination(pid, 1) = - particle_origin(pid, 1) + reflected_direction[1]; + particle_origin(pid, 1) + reflected_vector[1]; particle_destination(pid, 2) = - particle_origin(pid, 2) + reflected_direction[2]; + particle_origin(pid, 2) + reflected_vector[2]; } } }; From e43b841acbbd0420eab6f00b56657f7d707ec790 Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Wed, 29 Apr 2026 13:15:27 -0700 Subject: [PATCH 125/139] Enhance CI workflows with additional logging and configuration updates - Added logging of files to be checked in format-check workflow. - Updated static-analysis workflow to enable DEGAS2 support. - Changed build type to Debug and enabled DEGAS2 and tests in the test workflow. --- .github/workflows/format-check.yml | 5 +++++ .github/workflows/static-analysis.yml | 1 + .github/workflows/test.yml | 5 +++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml index 81cc715..f1ad8ca 100644 --- a/.github/workflows/format-check.yml +++ b/.github/workflows/format-check.yml @@ -31,6 +31,11 @@ jobs: exit 0 fi + echo "Files to be checked:" + for file in "${files[@]}"; do + echo " $file" + done + clang-format-20 --version clang-format-20 -style=file --dry-run -Werror "${files[@]}" diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index b1c6eb0..ba57e28 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -35,6 +35,7 @@ jobs: -DCMAKE_CXX_COMPILER=mpicxx \ -DCMAKE_C_COMPILER=mpicc \ -DBUILD_SHARED_LIBS=ON \ + -DPUMITALLY_ENABLE_DEGAS2=ON \ -DCMAKE_CXX_CLANG_TIDY=${TIDY_CMD} - name: Build with clang-tidy diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index df30e1e..5efa846 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,9 +26,10 @@ jobs: -DCMAKE_PREFIX_PATH=$DEPS_DIR \ -DCMAKE_CXX_COMPILER=mpicxx \ -DCMAKE_C_COMPILER=mpicc \ - -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_BUILD_TYPE=Debug \ -DBUILD_SHARED_LIBS=ON \ - -DPUMITALLYOPENMC_ENABLE_TESTS=ON + -DPUMITALLY_ENABLE_DEGAS2=ON \ + -DPUMITALLY_ENABLE_TESTS=ON - name: Build run: cmake --build build -j$(nproc) From f2bce18aedbb336f9932775647b7e0073e7a3310 Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Thu, 30 Apr 2026 09:42:27 -0700 Subject: [PATCH 126/139] Update CMakeLists.txt to link test_degas2_physics with pumipic library It is needed because it uses kokkos and other pumipic upstream libraries. All the linkings will be fixed later for coherence. --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 41b5a86..36bfc2a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -33,7 +33,7 @@ catch_discover_tests(test_pumi_tally_impl_methods) if(PUMITALLY_ENABLE_DEGAS2) add_executable(test_degas2_physics test_degas2_physics.cpp) target_include_directories(test_degas2_physics PRIVATE ${PROJECT_SOURCE_DIR}/src/physics/degas2) - target_link_libraries(test_degas2_physics PUBLIC Catch2::Catch2WithMain pumitally_degas2) + target_link_libraries(test_degas2_physics PUBLIC Catch2::Catch2WithMain pumitally_degas2 pumipic::pumipic) catch_discover_tests(test_degas2_physics DISCOVERY_MODE PRE_TEST) endif() #################### Test HostWrite #################### From 7afdf831d4785a652e17f0ce1c4777099a1bcaac Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Thu, 30 Apr 2026 09:47:38 -0700 Subject: [PATCH 127/139] Remove pull_request triggers from CI workflows This is to remove the double job dispatch in pull_requests. --- .github/workflows/format-check.yml | 2 -- .github/workflows/static-analysis.yml | 1 - .github/workflows/test.yml | 1 - 3 files changed, 4 deletions(-) diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml index f1ad8ca..5f81bc4 100644 --- a/.github/workflows/format-check.yml +++ b/.github/workflows/format-check.yml @@ -2,8 +2,6 @@ name: Format Check on: push: - pull_request: - pull_request_target: workflow_dispatch: jobs: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index ba57e28..eda84f5 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -2,7 +2,6 @@ name: Static Analysis on: push: - pull_request: workflow_dispatch: env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5efa846..c437a1b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,6 @@ name: Test on: push: - pull_request: workflow_dispatch: env: From 1154558af5a08914fb13cdb5624f6d2d19565098 Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Thu, 30 Apr 2026 09:57:36 -0700 Subject: [PATCH 128/139] Add LD_LIBRARY_PATH to CI workflow environment This shouldn't be necessary as it works locally and with spack without any problem. But in CI it says that it cannot find a pumipic shared library. It even worked in CI in main. ``` [ 92%] Linking CXX executable test_pumi_tally_impl_methods /home/runner/work/PumiTally/PumiTally/build/test/test_pumi_tally_impl_methods: error while loading shared libraries: libparticleStructs.so: cannot open shared object file: No such file or directory CMake Error at /home/runner/work/PumiTally/PumiTally/deps/lib/cmake/Catch2/CatchAddTests.cmake:70 (message): Error running test executable '/home/runner/work/PumiTally/PumiTally/build/test/test_pumi_tally_impl_methods': ``` --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c437a1b..d9658a2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,7 @@ on: env: DEPS_DIR: ${{ github.workspace }}/deps + LD_LIBRARY_PATH: ${{ github.workspace }}/deps/lib jobs: ctest: From 1440b872fe98104ba3ab1361f15fe9de0a7e9d22 Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Thu, 30 Apr 2026 10:02:46 -0700 Subject: [PATCH 129/139] Update static analysis workflow dependent on test's success - Changed trigger from push to workflow_run, depending on the completion of the Test workflow. - Added conditional execution for the clang-tidy job based on the event type and conclusion. - Updated checkout step to reference the correct commit SHA from the workflow run. --- .github/workflows/static-analysis.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index eda84f5..5d6fbfe 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -1,7 +1,10 @@ name: Static Analysis on: - push: + workflow_run: + workflows: ["Test"] + types: + - completed workflow_dispatch: env: @@ -10,9 +13,15 @@ env: jobs: clang-tidy: runs-on: ubuntu-latest + if: > + github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push') steps: - uses: actions/checkout@v6 + with: + ref: ${{ github.event.workflow_run.head_sha || github.ref }} - name: Install dependencies uses: ./.github/actions/install-deps From 2435668e1ee3e0dd51bc8734618dc57de2adf5ba Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Thu, 30 Apr 2026 10:09:50 -0700 Subject: [PATCH 130/139] add OpenMP environment variables before running tests --- .github/workflows/test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d9658a2..1fabd22 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,4 +35,8 @@ jobs: run: cmake --build build -j$(nproc) - name: Run tests - run: ctest --test-dir build --output-on-failure + run: | + export OMP_PROC_BIND=spread + export OMP_PLACES=threads + export OMP_NUM_THREADS=$(nproc) + ctest --test-dir build --output-on-failure From cdbaae7e38c36195767f86f93ac93ee2df81eeaa Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Thu, 30 Apr 2026 10:23:34 -0700 Subject: [PATCH 131/139] branch-specific triggers for main - Updated format-check and test workflows to trigger on push and pull_request events for the main branch. - Modified static-analysis workflow to simplify the conditional execution based on test workflow_run success. --- .github/workflows/format-check.yml | 3 +++ .github/workflows/static-analysis.yml | 3 +-- .github/workflows/test.yml | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml index 5f81bc4..d72e906 100644 --- a/.github/workflows/format-check.yml +++ b/.github/workflows/format-check.yml @@ -2,6 +2,9 @@ name: Format Check on: push: + branches: [main] + pull_request: + branches: [main] workflow_dispatch: jobs: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 5d6fbfe..fc0d707 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -15,8 +15,7 @@ jobs: runs-on: ubuntu-latest if: > github.event_name == 'workflow_dispatch' || - (github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.event == 'push') + (github.event.workflow_run.conclusion == 'success') steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1fabd22..72ef51a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,9 @@ name: Test on: push: + branches: [main] + pull_request: + branches: [main] workflow_dispatch: env: From 65cea8bb5f9f61cb763dee27c2ce5a7b492a0240 Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Thu, 30 Apr 2026 11:00:24 -0700 Subject: [PATCH 132/139] debug print particle locations --- test/test_degas2_physics.cpp | 47 ++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 8721874..8f6d744 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -8,6 +8,7 @@ #include "DG2Physics.h" #include #include +#include #include TEST_CASE("Test Degas2 Physics Functions") { @@ -98,20 +99,40 @@ TEST_CASE("Test Degas2 Physics Functions") { outfile << "Standard Deviation of Mean x Direction: " << sdux << std::endl; //0.579 - outfile << "Particle #,Velocity(m/s),Weight,X(cm),Y(cm),Z(cm),X_Dir,Y_Dir,Z_Dir" << std::endl; + outfile << std::left + << std::setw(10) << "Particle #" + << std::setw(16) << "Velocity(m/s)" + << std::setw(10) << "Weight" + << std::setw(10) << "X(cm)" + << std::setw(10) << "Y(cm)" + << std::setw(10) << "Z(cm)" + << std::setw(10) << "X_Dir" + << std::setw(10) << "Y_Dir" + << std::setw(10) << "Z_Dir" + << '\n'; + + for (int i = 0; i < numParticles; ++i) { + outfile << std::left + << std::setw(10) << i + << std::setw(16) << (1.0 / output(i).alpha) + << std::setw(10) << output(i).weight + << std::setw(10) << output(i).position[0] + << std::setw(10) << output(i).position[1] + << std::setw(10) << output(i).position[2] + << std::setw(10) << output(i).direction[0] + << std::setw(10) << output(i).direction[1] + << std::setw(10) << output(i).direction[2] + << '\n'; + } + + // print the file + std::ifstream infile("Log.txt"); + std::string line; + while (std::getline(infile, line)) { + std::cout << line << '\n'; + } + outfile.close(); - for (int i = 0; i < numParticles; ++i) { - outfile << i << ","; - outfile << (1.0/output(i).alpha) << ","; - outfile << output(i).weight << ","; - outfile << output(i).position[0] << ","; - outfile << output(i).position[1] << ","; - outfile << output(i).position[2] << ","; - outfile << output(i).direction[0] << ","; - outfile << output(i).direction[1] << ","; - outfile << output(i).direction[2] << std::endl; - } - REQUIRE_THAT(l, Catch::Matchers::WithinAbs(0.0240,0.0004)); REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(0.0240,0.0004)); REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); From d56922ae4a59fa621d30132cf3a53f2daccb2d30 Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Thu, 30 Apr 2026 11:43:32 -0700 Subject: [PATCH 133/139] update tolerance for position mean check - Initialized the variance variable `varux` to 0.0 . - Updated the tolerance for the test on variable `l`, `sdl` to 0.0015, reflecting a more accurate confidence interval for 1000 samples. --- test/test_degas2_physics.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 8f6d744..0672448 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -88,7 +88,7 @@ TEST_CASE("Test Degas2 Physics Functions") { ux /= numParticles; outfile << "Mean x Direction: " << ux << std::endl; //0.0159 - double varux; + double varux {0.0}; for (int i=0; i < numParticles; ++i) { varux += (output(i).direction[0] - ux)*(output(i).direction[0] - ux); } @@ -133,8 +133,13 @@ TEST_CASE("Test Degas2 Physics Functions") { } outfile.close(); - REQUIRE_THAT(l, Catch::Matchers::WithinAbs(0.0240,0.0004)); - REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(0.0240,0.0004)); + // Fixme: Where did this tolerance come from. Looks like it's + // not enough for 1000 particles. + // Tolerance is 0.0015 for l, with 95% confidence interval for mean + // with 1000 samples. But we should use something like 2-3 standard deviations + // with more than 99% confidence. + REQUIRE_THAT(l, Catch::Matchers::WithinAbs(0.0240,0.0015)); + REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(0.0240,0.0015)); REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); } From 570b3252bcb6e080bde4113cc9d7ed2aa5796fd6 Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Thu, 30 Apr 2026 11:58:38 -0700 Subject: [PATCH 134/139] format degas2 physics changes --- src/physics/degas2/DG2Physics.h | 421 ++++++++++++++++++----------- src/physics/degas2/degas2-main.cpp | 187 +++++++------ src/pumitally/PumiTallyImpl.cpp | 48 ++-- src/pumitally/PumiTallyImpl.h | 5 +- test/test_degas2_physics.cpp | 176 ++++++------ 5 files changed, 464 insertions(+), 373 deletions(-) diff --git a/src/physics/degas2/DG2Physics.h b/src/physics/degas2/DG2Physics.h index e1549ae..336d2c2 100644 --- a/src/physics/degas2/DG2Physics.h +++ b/src/physics/degas2/DG2Physics.h @@ -20,7 +20,7 @@ struct ParticleInfo { double weight; int energy_group; // Energy group *index* int particle_index; - double alpha; //To be multiplied in when computing tally + double alpha; // To be multiplied in when computing tally }; struct FieldInfo { @@ -37,9 +37,9 @@ class DG2Physics { const int seed = SEED) : random_pool(seed), cross_section(cross_section_file) { // Initialize particle energy - particle_energy = - Kokkos::View("particle_energy", num_particles); - //cross_sections = Kokkos::View("cross_sections", num_particles); + particle_energy = Kokkos::View("particle_energy", num_particles); + // cross_sections = Kokkos::View("cross_sections", + // num_particles); } DG2Physics(const DG2CrossSection cross_section, const int num_particles, @@ -47,120 +47,202 @@ class DG2Physics { : random_pool(seed), cross_section(cross_section) { // Initialize particle energy - particle_energy = - Kokkos::View("particle_energy", num_particles); - //cross_sections = Kokkos::View("cross_sections", num_particles); + particle_energy = Kokkos::View("particle_energy", num_particles); + // cross_sections = Kokkos::View("cross_sections", + // num_particles); } KOKKOS_FUNCTION - void set_energy(ParticleInfo &particle_info, const double energy) const{ - particle_energy(particle_info.particle_index) = energy; + void set_energy(ParticleInfo &particle_info, const double energy) const { + particle_energy(particle_info.particle_index) = energy; } KOKKOS_FUNCTION - double ionization_cross_section(ParticleInfo &particle_info, const FieldInfo &field_info) const{ - double energy = particle_energy(particle_info.particle_index); - double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 + double ionization_cross_section(ParticleInfo &particle_info, + const FieldInfo &field_info) const { + double energy = particle_energy(particle_info.particle_index); + double mp{938.27e6 / (3e10 * 3e10)}; // eV/c^2 = eV*s^2/cm^2 + double particle_velocity_squared{2 * energy / mp}; // cm^2/s^2 + + // Compute Ionization Cross Section + double coef2[9]; + + coef2[0] = -3.271396786375e1; + coef2[1] = 1.353655609057e1; + coef2[2] = -5.739328757388; + coef2[3] = 1.563154982022; + coef2[4] = -2.877056004391e-1; + coef2[5] = 3.482559773737e-2; + coef2[6] = -2.631976175590e-3; + coef2[7] = 1.119543953861e-4; + coef2[8] = -2.039149852002e-6; + + double lnrate_ion{0}; // cm^2 - //Compute Ionization Cross Section - double coef2 [9]; - - coef2[0] = -3.271396786375e1; coef2[1] = 1.353655609057e1; coef2[2] = -5.739328757388; - coef2[3] = 1.563154982022; coef2[4] = -2.877056004391e-1; coef2[5] = 3.482559773737e-2; - coef2[6] = -2.631976175590e-3; coef2[7] = 1.119543953861e-4; coef2[8] = -2.039149852002e-6; - - double lnrate_ion {0}; //cm^2 - - for (int i=0; i<9; i++) { - lnrate_ion += coef2[i]*Kokkos::pow(Kokkos::log(field_info.electron_temperature),i); + for (int i = 0; i < 9; i++) { + lnrate_ion += + coef2[i] * + Kokkos::pow(Kokkos::log(field_info.electron_temperature), i); } - double sigma_ion = Kokkos::exp(lnrate_ion)/Kokkos::sqrt(particle_velocity_squared); + double sigma_ion = + Kokkos::exp(lnrate_ion) / Kokkos::sqrt(particle_velocity_squared); return sigma_ion; } KOKKOS_FUNCTION - double charge_exchange_cross_section(ParticleInfo &particle_info, const FieldInfo &field_info) const{ - double energy = particle_energy(particle_info.particle_index); - double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double particle_velocity_squared {2*energy/mp}; //cm^2/s^2 - - //Compute Charge Exchange Cross Section - double coef [9][9]; //E index, T index - - coef[0][0] = -1.829079581680e1; coef[0][1] = 2.169137615703e-1; coef[0][2] = 4.307131243894e-2; - coef[0][3] = -5.754895093075e-4; coef[0][4] = -1.552077120204e-3; coef[0][5] = -1.876800283030e-4; - coef[0][6] = 1.125490270962e-4; coef[0][7] = -1.238982763007e-5; coef[0][8] = 4.163596197181e-07; - - coef[1][0] = 1.640252721210e-1; coef[1][1] = -1.106722014459e-1; coef[1][2] = 8.948693624917e-3; - coef[1][3] = 6.062141761233e-3; coef[1][4] = -1.210431587568e-3; coef[1][5] = -4.052878751584e-5; - coef[1][6] = 2.875900435895e-5; coef[1][7] = -2.616998139678e-6; coef[1][8] = 7.558092849125e-8; - - coef[2][0] = 3.364564509137e-2; coef[2][1] = -1.382158680424e-3; coef[2][2] = -1.209480567154e-02; - coef[2][3] = 1.075907881928e-3; coef[2][4] = 8.297212635856e-4; coef[2][5] = -1.907025662962e-04; - coef[2][6] = 1.338839628570e-5; coef[2][7] = -1.171762874107e-7; coef[2][8] = -1.328404104165e-8; - - coef[3][0] = 9.530225559189e-3; coef[3][1] = 7.348786286628e-3; coef[3][2] = -3.675019470470e-4; - coef[3][3] = -8.119301728339e-4; coef[3][4] = 1.361661816974e-4; coef[3][5] = 1.141663041636e-5; - coef[3][6] = -4.340802793033e-6; coef[3][7] = 3.517971869029e-7; coef[3][8] = -9.170850253981e-09; - - coef[4][0] = -8.519413589968e-4; coef[4][1] = -6.343059502294e-4; coef[4][2] = 1.039643390686e-3; - coef[4][3] = 8.911036876068e-6; coef[4][4] = -1.008928628425e-4; coef[4][5] = 1.775681984457e-05; - coef[4][6] = -7.003521917385e-7; coef[4][7] = -4.928692832866e-8; coef[4][8] = 3.208853883734e-9; - - coef[5][0] = -1.247583860943e-3; coef[5][1] = -1.919569450380e-4; coef[5][2] = -1.553840717902e-4; - coef[5][3] = 3.175388949811e-5; coef[5][4] = 1.080693990468e-5; coef[5][5] = -3.149286923815e-6; - coef[5][6] = 2.318308730487e-7; coef[5][7] = 1.756388998863e-10; coef[5][8] = -3.952740758950e-10; - - coef[6][0] = 3.014307545716e-4; coef[6][1] = 4.075019351738e-5; coef[6][2] = 2.670827249272e-6; - coef[6][3] = -4.515123641755e-6; coef[6][4] = 5.106059413591e-7; coef[6][5] = 3.105491554749e-8; - coef[6][6] = -6.030983538280e-9; coef[6][7] = -1.446756795654e-10; coef[6][8] = 2.739558475782e-11; - - coef[7][0] = -2.499323170044e-5; coef[7][1] = -2.850044983009e-6; coef[7][2] = 7.695300597935e-7; - coef[7][3] = 2.187439283954e-7; coef[7][4] = -1.299275586093e-7; coef[7][5] = 2.274394089017e-8; - coef[7][6] = -1.755944926274e-9; coef[7][7] = 7.143183138281e-11; coef[7][8] = -1.693040208927e-12; - - coef[8][0] = 6.932627237765e-7; coef[8][1] = 6.966822400446e-8; coef[8][2] = -3.783302281524e-8; - coef[8][3] = -2.911233951880e-9; coef[8][4] = 5.117133050290e-9; coef[8][5] = -1.130988250912e-9; - coef[8][6] = 1.005189187279e-10; coef[8][7] = -3.989884105603e-12; coef[8][8] = 6.388219930167e-14; - - double lnrate_cx {0}; + double charge_exchange_cross_section(ParticleInfo &particle_info, + const FieldInfo &field_info) const { + double energy = particle_energy(particle_info.particle_index); + double mp{938.27e6 / (3e10 * 3e10)}; // eV/c^2 = eV*s^2/cm^2 + double particle_velocity_squared{2 * energy / mp}; // cm^2/s^2 + + // Compute Charge Exchange Cross Section + double coef[9][9]; // E index, T index + + coef[0][0] = -1.829079581680e1; + coef[0][1] = 2.169137615703e-1; + coef[0][2] = 4.307131243894e-2; + coef[0][3] = -5.754895093075e-4; + coef[0][4] = -1.552077120204e-3; + coef[0][5] = -1.876800283030e-4; + coef[0][6] = 1.125490270962e-4; + coef[0][7] = -1.238982763007e-5; + coef[0][8] = 4.163596197181e-07; + + coef[1][0] = 1.640252721210e-1; + coef[1][1] = -1.106722014459e-1; + coef[1][2] = 8.948693624917e-3; + coef[1][3] = 6.062141761233e-3; + coef[1][4] = -1.210431587568e-3; + coef[1][5] = -4.052878751584e-5; + coef[1][6] = 2.875900435895e-5; + coef[1][7] = -2.616998139678e-6; + coef[1][8] = 7.558092849125e-8; + + coef[2][0] = 3.364564509137e-2; + coef[2][1] = -1.382158680424e-3; + coef[2][2] = -1.209480567154e-02; + coef[2][3] = 1.075907881928e-3; + coef[2][4] = 8.297212635856e-4; + coef[2][5] = -1.907025662962e-04; + coef[2][6] = 1.338839628570e-5; + coef[2][7] = -1.171762874107e-7; + coef[2][8] = -1.328404104165e-8; + + coef[3][0] = 9.530225559189e-3; + coef[3][1] = 7.348786286628e-3; + coef[3][2] = -3.675019470470e-4; + coef[3][3] = -8.119301728339e-4; + coef[3][4] = 1.361661816974e-4; + coef[3][5] = 1.141663041636e-5; + coef[3][6] = -4.340802793033e-6; + coef[3][7] = 3.517971869029e-7; + coef[3][8] = -9.170850253981e-09; + + coef[4][0] = -8.519413589968e-4; + coef[4][1] = -6.343059502294e-4; + coef[4][2] = 1.039643390686e-3; + coef[4][3] = 8.911036876068e-6; + coef[4][4] = -1.008928628425e-4; + coef[4][5] = 1.775681984457e-05; + coef[4][6] = -7.003521917385e-7; + coef[4][7] = -4.928692832866e-8; + coef[4][8] = 3.208853883734e-9; + + coef[5][0] = -1.247583860943e-3; + coef[5][1] = -1.919569450380e-4; + coef[5][2] = -1.553840717902e-4; + coef[5][3] = 3.175388949811e-5; + coef[5][4] = 1.080693990468e-5; + coef[5][5] = -3.149286923815e-6; + coef[5][6] = 2.318308730487e-7; + coef[5][7] = 1.756388998863e-10; + coef[5][8] = -3.952740758950e-10; + + coef[6][0] = 3.014307545716e-4; + coef[6][1] = 4.075019351738e-5; + coef[6][2] = 2.670827249272e-6; + coef[6][3] = -4.515123641755e-6; + coef[6][4] = 5.106059413591e-7; + coef[6][5] = 3.105491554749e-8; + coef[6][6] = -6.030983538280e-9; + coef[6][7] = -1.446756795654e-10; + coef[6][8] = 2.739558475782e-11; + + coef[7][0] = -2.499323170044e-5; + coef[7][1] = -2.850044983009e-6; + coef[7][2] = 7.695300597935e-7; + coef[7][3] = 2.187439283954e-7; + coef[7][4] = -1.299275586093e-7; + coef[7][5] = 2.274394089017e-8; + coef[7][6] = -1.755944926274e-9; + coef[7][7] = 7.143183138281e-11; + coef[7][8] = -1.693040208927e-12; + + coef[8][0] = 6.932627237765e-7; + coef[8][1] = 6.966822400446e-8; + coef[8][2] = -3.783302281524e-8; + coef[8][3] = -2.911233951880e-9; + coef[8][4] = 5.117133050290e-9; + coef[8][5] = -1.130988250912e-9; + coef[8][6] = 1.005189187279e-10; + coef[8][7] = -3.989884105603e-12; + coef[8][8] = 6.388219930167e-14; + + double lnrate_cx{0}; for (int i = 0; i < 9; i++) { - for (int j = 0; j < 9; j++) { - lnrate_cx += coef[i][j]*Kokkos::pow(Kokkos::log(energy),i)*Kokkos::pow(Kokkos::log(field_info.ion_temperature),j); - } + for (int j = 0; j < 9; j++) { + lnrate_cx += coef[i][j] * Kokkos::pow(Kokkos::log(energy), i) * + Kokkos::pow(Kokkos::log(field_info.ion_temperature), j); + } } - double sigma_cx = Kokkos::exp(lnrate_cx)/Kokkos::sqrt(particle_velocity_squared); - return sigma_cx; + double sigma_cx = + Kokkos::exp(lnrate_cx) / Kokkos::sqrt(particle_velocity_squared); + return sigma_cx; } - KOKKOS_FUNCTION //IF THIS IS CHANGED MAY NEED TO CHANGE max_sigma_cx in the collision code - double analytic_cross_section_CE(double energy) const{ - return (0.6937e-14*(1-0.155*Kokkos::log10(energy))*(1-0.155*Kokkos::log10(energy)))/(1+0.1112e-14*Kokkos::pow(energy,3.3)); + KOKKOS_FUNCTION // IF THIS IS CHANGED MAY NEED TO CHANGE max_sigma_cx in the + // collision code + double + analytic_cross_section_CE(double energy) const { + return (0.6937e-14 * (1 - 0.155 * Kokkos::log10(energy)) * + (1 - 0.155 * Kokkos::log10(energy))) / + (1 + 0.1112e-14 * Kokkos::pow(energy, 3.3)); } KOKKOS_FUNCTION void sample_collision_distance(ParticleInfo &particle_info, const FieldInfo &field_info) const { - // example of sampling a random number + // example of sampling a random number auto rand_gen = random_pool.get_state(); double x = rand_gen.drand(0., 1.); random_pool.free_state(rand_gen); - double sigma_ion = ionization_cross_section(particle_info, field_info); - double sigma_cx = charge_exchange_cross_section(particle_info, field_info); - - //Generate distance and move particle -// printf("\n$$$ Before moving particle [%d], position: (%f, %f, %f) $$$\n", particle_info.particle_index, particle_info.position[0], particle_info.position[1], -// particle_info.position[2]); - double l =-Kokkos::log(x)/(field_info.electron_density*sigma_ion+field_info.ion_density*sigma_cx)*0.01; //m. n in cm^-3 - particle_info.position[0] += l*particle_info.direction[0]; - particle_info.position[1] += l*particle_info.direction[1]; - particle_info.position[2] += l*particle_info.direction[2]; -// printf("\n$$$ After moving particle [%d], position (%f, %f, %f). Direction was (%f, %f, %f) with l = %f $$$\n", particle_info.particle_index, particle_info.position[0], -// particle_info.position[1], particle_info.position[2], particle_info.direction[0], particle_info.direction[1], particle_info.direction[2], l); - double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - particle_info.alpha = Kokkos::sqrt(mp/(2.0*particle_energy(particle_info.particle_index)))*100; //1/(m/s) + double sigma_ion = ionization_cross_section(particle_info, field_info); + double sigma_cx = charge_exchange_cross_section(particle_info, field_info); + + // Generate distance and move particle + // printf("\n$$$ Before moving particle [%d], position: (%f, %f, %f) + //$$$\n", particle_info.particle_index, particle_info.position[0], + // particle_info.position[1], particle_info.position[2]); + double l = -Kokkos::log(x) / + (field_info.electron_density * sigma_ion + + field_info.ion_density * sigma_cx) * + 0.01; // m. n in cm^-3 + particle_info.position[0] += l * particle_info.direction[0]; + particle_info.position[1] += l * particle_info.direction[1]; + particle_info.position[2] += l * particle_info.direction[2]; + // printf("\n$$$ After moving particle [%d], position (%f, %f, %f). + // Direction was (%f, %f, %f) with l = %f $$$\n", + // particle_info.particle_index, particle_info.position[0], + // particle_info.position[1], particle_info.position[2], + // particle_info.direction[0], particle_info.direction[1], + // particle_info.direction[2], l); + double mp{938.27e6 / (3e10 * 3e10)}; // eV/c^2 = eV*s^2/cm^2 + particle_info.alpha = + Kokkos::sqrt(mp / + (2.0 * particle_energy(particle_info.particle_index))) * + 100; // 1/(m/s) } // collision event @@ -168,85 +250,96 @@ class DG2Physics { void collide_particle(ParticleInfo &particle_info, const FieldInfo &field_info) const { - //Generate Random Numbers - auto rand_gen = random_pool.get_state(); - double ww = rand_gen.drand(0., 1.); - - double sigma_ion = ionization_cross_section(particle_info, field_info); - double sigma_cx = charge_exchange_cross_section(particle_info, field_info); - - //Compute New Direction and Energy and set particle info - //Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) - double mp {938.27e6/(3e10*3e10)}; //eV/c^2 = eV*s^2/cm^2 - double max_rate_cx = 1.4e-7; - - bool rejection_test = false; - auto old_mag_v = Kokkos::sqrt(2*particle_energy(particle_info.particle_index)/mp); - double vx {}; - double vy {}; - double vz {}; - //Loops this until it passes the rejection test + // Generate Random Numbers + auto rand_gen = random_pool.get_state(); + double ww = rand_gen.drand(0., 1.); + + double sigma_ion = ionization_cross_section(particle_info, field_info); + double sigma_cx = charge_exchange_cross_section(particle_info, field_info); + + // Compute New Direction and Energy and set particle info + // Compute 3 Maxwellian (Gaussian) distributed velocities (cm/s) + double mp{938.27e6 / (3e10 * 3e10)}; // eV/c^2 = eV*s^2/cm^2 + double max_rate_cx = 1.4e-7; + + bool rejection_test = false; + auto old_mag_v = + Kokkos::sqrt(2 * particle_energy(particle_info.particle_index) / mp); + double vx{}; + double vy{}; + double vz{}; + // Loops this until it passes the rejection test while (!rejection_test) { - //Generate random numbers for the velocity - double x1 = rand_gen.drand(0., 1.); - double x2 = rand_gen.drand(0., 1.); - double y1 = rand_gen.drand(0., 1.); - double y2 = rand_gen.drand(0., 1.); - - //Sample a new velocity from the thermal distribution (cm/s) - vx = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::cos(2*M_PI*x2); - vy = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(x1))*Kokkos::sin(2*M_PI*x2); - vz = Kokkos::sqrt(field_info.ion_temperature/mp)*Kokkos::sqrt(-2 * Kokkos::log(y1))*Kokkos::sin(2*M_PI*y2); - - //Compute the relative velocity (note using this isn't thoroughly tested. Can be disabled by changing - //old_mag_v to 0, then it is just vx, vy, vz like I originally used. - auto rel_vx = vx - particle_info.direction[0]*old_mag_v; - auto rel_vy = vy - particle_info.direction[1]*old_mag_v; - auto rel_vz = vz - particle_info.direction[2]*old_mag_v; - auto mag_v2 = rel_vx*rel_vx + rel_vy*rel_vy + rel_vz*rel_vz; - - //Generate random number and compare to sigma/sigma_max - if (rand_gen.drand(0., 1.) < Kokkos::sqrt(mag_v2)*analytic_cross_section_CE(0.5*mp*mag_v2)/max_rate_cx) { - rejection_test = true; - } + // Generate random numbers for the velocity + double x1 = rand_gen.drand(0., 1.); + double x2 = rand_gen.drand(0., 1.); + double y1 = rand_gen.drand(0., 1.); + double y2 = rand_gen.drand(0., 1.); + + // Sample a new velocity from the thermal distribution (cm/s) + vx = Kokkos::sqrt(field_info.ion_temperature / mp) * + Kokkos::sqrt(-2 * Kokkos::log(x1)) * Kokkos::cos(2 * M_PI * x2); + vy = Kokkos::sqrt(field_info.ion_temperature / mp) * + Kokkos::sqrt(-2 * Kokkos::log(x1)) * Kokkos::sin(2 * M_PI * x2); + vz = Kokkos::sqrt(field_info.ion_temperature / mp) * + Kokkos::sqrt(-2 * Kokkos::log(y1)) * Kokkos::sin(2 * M_PI * y2); + + // Compute the relative velocity (note using this isn't thoroughly tested. + // Can be disabled by changing old_mag_v to 0, then it is just vx, vy, vz + // like I originally used. + auto rel_vx = vx - particle_info.direction[0] * old_mag_v; + auto rel_vy = vy - particle_info.direction[1] * old_mag_v; + auto rel_vz = vz - particle_info.direction[2] * old_mag_v; + auto mag_v2 = rel_vx * rel_vx + rel_vy * rel_vy + rel_vz * rel_vz; + + // Generate random number and compare to sigma/sigma_max + if (rand_gen.drand(0., 1.) < + Kokkos::sqrt(mag_v2) * analytic_cross_section_CE(0.5 * mp * mag_v2) / + max_rate_cx) { + rejection_test = true; + } } - //Now that the rejection test has passed, set the new information - double mag_v{Kokkos::sqrt(vx*vx+vy*vy+vz*vz)}; - particle_info.direction[0] = vx/mag_v; - particle_info.direction[1] = vy/mag_v; - particle_info.direction[2] = vz/mag_v; - - particle_energy(particle_info.particle_index) = 0.5*mp*mag_v*mag_v; - //particle_info.energy_group = particle_energy(particle_info.particle_index); //Temporary for debugging - //Adjust Weights - double new_weight = particle_info.weight*(1- - (field_info.electron_density*sigma_ion)/ - (field_info.ion_density*sigma_cx+field_info.electron_density*sigma_ion)); - - //Russian Roulette - double wc = 0.25; - double ws = 1.0; - - if (new_weight < wc){ - if (ww<(1-new_weight/ws)) { - new_weight = 0; - } - else { - new_weight = ws; - } + // Now that the rejection test has passed, set the new information + double mag_v{Kokkos::sqrt(vx * vx + vy * vy + vz * vz)}; + particle_info.direction[0] = vx / mag_v; + particle_info.direction[1] = vy / mag_v; + particle_info.direction[2] = vz / mag_v; + + particle_energy(particle_info.particle_index) = 0.5 * mp * mag_v * mag_v; + // particle_info.energy_group = + // particle_energy(particle_info.particle_index); //Temporary for debugging + // Adjust Weights + double new_weight = particle_info.weight * + (1 - (field_info.electron_density * sigma_ion) / + (field_info.ion_density * sigma_cx + + field_info.electron_density * sigma_ion)); + + // Russian Roulette + double wc = 0.25; + double ws = 1.0; + + if (new_weight < wc) { + if (ww < (1 - new_weight / ws)) { + new_weight = 0; + } else { + new_weight = ws; + } } - particle_info.weight = new_weight; - - //This definition may be redundant since it is also in the next location function - particle_info.alpha = Kokkos::sqrt(mp/(2.0*particle_energy(particle_info.particle_index)))*100; //1/(m/s) - random_pool.free_state(rand_gen); + particle_info.weight = new_weight; + + // This definition may be redundant since it is also in the next location + // function + particle_info.alpha = + Kokkos::sqrt(mp / + (2.0 * particle_energy(particle_info.particle_index))) * + 100; // 1/(m/s) + random_pool.free_state(rand_gen); } -//To here + // To here random_pool_t random_pool; DG2CrossSection cross_section; Kokkos::View particle_energy; - }; #endif // PUMITALLYOPENMC_DG2PHYSICS_H diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index b944887..bfa77b4 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -27,9 +27,9 @@ void print_initial_info(const std::string &mesh_name, // computes centroids of each element in the mesh and stores them in a tag void get_centroids(Omega_h::Mesh &mesh, Omega_h::Write centroids); -void -sample_initial_particle_energy_direction(Kokkos::View energy_array, Kokkos::View direction, - Kokkos::View alpha); +void sample_initial_particle_energy_direction( + Kokkos::View energy_array, Kokkos::View direction, + Kokkos::View alpha); template void vector2write(const std::vector &vec, Omega_h::Write &write_vec) { @@ -52,67 +52,73 @@ void set_field_values_to_mesh(Omega_h::Mesh &mesh, const Fields &fields); // void set_source_particles(Omega_h::Mesh &mesh, // SourceDistribution source_distribution, // Omega_h::Write particle_positions); -void -transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, const Omega_h::Read &electron_density, - const Omega_h::Read ion_density, const Omega_h::Read electron_temperature, - const Omega_h::Read ion_temperature, const Omega_h::Read bulk_flow_velocity, - const Kokkos::View initial_direction, const Kokkos::View initial_alpha, - int max_iterations); +void transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, + const Omega_h::Read &electron_density, + const Omega_h::Read ion_density, + const Omega_h::Read electron_temperature, + const Omega_h::Read ion_temperature, + const Omega_h::Read bulk_flow_velocity, + const Kokkos::View initial_direction, + const Kokkos::View initial_alpha, int max_iterations); // ***************************************************************************// // *************************** Main Function ******************************** // // ***************************************************************************// int main(int argc, char *argv[]) { - Kokkos::initialize(argc, argv); - { - // Read input parameters - InputParameters input_params; - read_input_parameters(argc, argv, input_params); - print_initial_info(input_params.mesh_name, input_params.num_particles); - - // Initialize PUMI-Tally and Read Fields - auto pumi_tally = pumitally::PumiTallyImpl( - input_params.mesh_name, input_params.num_particles, argc, argv, input_params.source_distribution); - auto &mesh = pumi_tally.full_mesh; - Omega_h::Write centroids(mesh.nelems() * 3); - get_centroids(mesh, centroids); - Fields fields; - get_field_values(Omega_h::Reals(centroids), fields); - set_field_values_to_mesh(mesh, fields); - pumi_tally.is_pumipic_initialized = true; - - Kokkos::View < Omega_h::Real * ** > sigma_t_; // mat, T, g - Kokkos::View < Omega_h::Real * ** > sigma_a_; // mat, T, g - Kokkos::View < Omega_h::Real * *** > scattering_matrix_; // mat, T, g, g - Kokkos::View < Omega_h::Real * ** > sigma_s_; // mat, T, g - DG2CrossSection crossSection(sigma_t_, sigma_a_, scattering_matrix_, - sigma_s_); - DG2Physics physics(crossSection, input_params.num_particles); - - Kokkos::View initial_direction("Initial Direction", input_params.num_particles * 3); - Kokkos::View initial_alpha("Initial Alpha", input_params.num_particles); - sample_initial_particle_energy_direction(physics.particle_energy, initial_direction, initial_alpha); - - // Move particles - auto electron_density = - mesh.get_array(Omega_h::REGION, "electron_density"); - auto ion_density = - mesh.get_array(Omega_h::REGION, "ion_density"); - auto electron_temperature = - mesh.get_array(Omega_h::REGION, "electron_temperature"); - auto ion_temperature = - mesh.get_array(Omega_h::REGION, "ion_temperature"); - auto bulk_flow_velocity = - mesh.get_array(Omega_h::REGION, "bulk_flow_velocity"); - - transport(pumi_tally, physics, electron_density, ion_density, - electron_temperature, ion_temperature, bulk_flow_velocity, initial_direction, - initial_alpha, input_params.max_iterations); - - // Finalize and output results - pumi_tally.WriteTallyResults(); - } - Kokkos::finalize(); + Kokkos::initialize(argc, argv); + { + // Read input parameters + InputParameters input_params; + read_input_parameters(argc, argv, input_params); + print_initial_info(input_params.mesh_name, input_params.num_particles); + + // Initialize PUMI-Tally and Read Fields + auto pumi_tally = pumitally::PumiTallyImpl( + input_params.mesh_name, input_params.num_particles, argc, argv, + input_params.source_distribution); + auto &mesh = pumi_tally.full_mesh; + Omega_h::Write centroids(mesh.nelems() * 3); + get_centroids(mesh, centroids); + Fields fields; + get_field_values(Omega_h::Reals(centroids), fields); + set_field_values_to_mesh(mesh, fields); + pumi_tally.is_pumipic_initialized = true; + + Kokkos::View sigma_t_; // mat, T, g + Kokkos::View sigma_a_; // mat, T, g + Kokkos::View scattering_matrix_; // mat, T, g, g + Kokkos::View sigma_s_; // mat, T, g + DG2CrossSection crossSection(sigma_t_, sigma_a_, scattering_matrix_, + sigma_s_); + DG2Physics physics(crossSection, input_params.num_particles); + + Kokkos::View initial_direction("Initial Direction", + input_params.num_particles * 3); + Kokkos::View initial_alpha("Initial Alpha", + input_params.num_particles); + sample_initial_particle_energy_direction(physics.particle_energy, + initial_direction, initial_alpha); + + // Move particles + auto electron_density = + mesh.get_array(Omega_h::REGION, "electron_density"); + auto ion_density = + mesh.get_array(Omega_h::REGION, "ion_density"); + auto electron_temperature = + mesh.get_array(Omega_h::REGION, "electron_temperature"); + auto ion_temperature = + mesh.get_array(Omega_h::REGION, "ion_temperature"); + auto bulk_flow_velocity = + mesh.get_array(Omega_h::REGION, "bulk_flow_velocity"); + + transport(pumi_tally, physics, electron_density, ion_density, + electron_temperature, ion_temperature, bulk_flow_velocity, + initial_direction, initial_alpha, input_params.max_iterations); + + // Finalize and output results + pumi_tally.WriteTallyResults(); + } + Kokkos::finalize(); return 0; } @@ -124,12 +130,14 @@ int main(int argc, char *argv[]) { // **************************************************************************************************************** // // -void -transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, const Omega_h::Read &electron_density, - const Omega_h::Read ion_density, const Omega_h::Read electron_temperature, - const Omega_h::Read ion_temperature, const Omega_h::Read bulk_flow_velocity, - const Kokkos::View initial_direction, const Kokkos::View initial_alpha, - int max_iterations) { +void transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, + const Omega_h::Read &electron_density, + const Omega_h::Read ion_density, + const Omega_h::Read electron_temperature, + const Omega_h::Read ion_temperature, + const Omega_h::Read bulk_flow_velocity, + const Kokkos::View initial_direction, + const Kokkos::View initial_alpha, int max_iterations) { for (int iter = 0; iter < max_iterations; ++iter) { auto particle_dest = pumi_tally.pumipic_ptcls->get<1>(); @@ -139,8 +147,8 @@ transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, const Omega auto last_exit = pumi_tally.p_pumi_particle_at_elem_boundary_handler->last_exit_; - - printf("\n### iter = %d ###\n",iter); + + printf("\n### iter = %d ###\n", iter); auto alpha = pumi_tally.p_pumi_particle_at_elem_boundary_handler->alpha_; auto get_new_destination = @@ -148,7 +156,7 @@ transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, const Omega if (mask > 0) { // FIXME: check if the particle is at destination or at // the boundary - // + // ParticleInfo particle_info; particle_info.position[0] = particle_orig(pid, 0); particle_info.position[1] = particle_orig(pid, 1); @@ -159,13 +167,13 @@ transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, const Omega particle_info.direction[1] = initial_direction(pid * 3 + 1); particle_info.direction[2] = initial_direction(pid * 3 + 2); } else { - auto direction = Omega_h::normalize(Omega_h::Vector < 3 > { - particle_info.position[0] - particle_dest(pid, 0), - particle_info.position[1] - particle_dest(pid, 1), - particle_info.position[2] - particle_dest(pid, 2)}); - particle_info.direction[0] = direction[0]; - particle_info.direction[1] = direction[1]; - particle_info.direction[2] = direction[2]; + auto direction = Omega_h::normalize(Omega_h::Vector<3>{ + particle_info.position[0] - particle_dest(pid, 0), + particle_info.position[1] - particle_dest(pid, 1), + particle_info.position[2] - particle_dest(pid, 2)}); + particle_info.direction[0] = direction[0]; + particle_info.direction[1] = direction[1]; + particle_info.direction[2] = direction[2]; } particle_info.weight = particle_weight(pid); particle_info.energy_group = particle_group(pid); @@ -179,19 +187,22 @@ transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, const Omega field_info.bulk_flow_velocity[0] = bulk_flow_velocity[e * 3 + 0]; field_info.bulk_flow_velocity[1] = bulk_flow_velocity[e * 3 + 1]; field_info.bulk_flow_velocity[2] = bulk_flow_velocity[e * 3 + 2]; - - -// printf("\n---- Direction (pid = %d) before collision: (%f, %f, %f). Original pos: (%f, %f, %f), pos (%f, %f, %f) ---- \n", pid, direction[0], direction[1], direction[2], -// particle_orig(pid,0), particle_orig(pid,1), particle_orig(pid,2), particle_info.position[0], particle_info.position[1], particle_info.position[2]); + + // printf("\n---- Direction (pid = %d) before collision: (%f, %f, + //%f). Original pos: (%f, %f, %f), pos (%f, %f, %f) ---- \n", pid, + // direction[0], direction[1], direction[2], + // particle_orig(pid,0), particle_orig(pid,1), particle_orig(pid,2), + // particle_info.position[0], particle_info.position[1], + // particle_info.position[2]); if (iter != 0) { // first iteration, we do not need to collide if (last_exit[pid] == -1) { // reached destination physics.collide_particle(particle_info, field_info); } - } + } physics.sample_collision_distance(particle_info, field_info); - + // Update particle position and direction particle_dest(pid, 0) = particle_info.position[0]; particle_dest(pid, 1) = particle_info.position[1]; @@ -357,7 +368,7 @@ void get_field_values(Omega_h::Reals centroids, Fields &fields) { fields.ion_temperature.resize(num_elements, 1.0); fields.electron_density.resize(num_elements, 1.0); fields.ion_density.resize(num_elements, 1.0); - fields.bulk_flow_velocity.resize(3*num_elements, 0.0); + fields.bulk_flow_velocity.resize(3 * num_elements, 0.0); // TODO: Add your code here to retrieve the actual field values // Write centroid coords to CSV to be read by python script @@ -432,9 +443,9 @@ void read_input_parameters(int argc, char *const *argv, } } -void -sample_initial_particle_energy_direction(Kokkos::View energy_array, Kokkos::View direction, - Kokkos::View alpha) { +void sample_initial_particle_energy_direction( + Kokkos::View energy_array, Kokkos::View direction, + Kokkos::View alpha) { random_pool_t randomPool(0); auto sample_energy = OMEGA_H_LAMBDA(int i) { // Basically, feed this 4 uniformly generated random numbers, x1, x2, y1, @@ -487,9 +498,7 @@ sample_initial_particle_energy_direction(Kokkos::View energy_array, Ko // computations. double particle_energy = 0.5 * mp * mag_v * mag_v; energy_array(i) = particle_energy; - - }; - Kokkos::parallel_for("sample_initial_particle_energy_direction", energy_array.size(), - sample_energy); + Kokkos::parallel_for("sample_initial_particle_energy_direction", + energy_array.size(), sample_energy); } diff --git a/src/pumitally/PumiTallyImpl.cpp b/src/pumitally/PumiTallyImpl.cpp index 5af1592..fc0bdc0 100644 --- a/src/pumitally/PumiTallyImpl.cpp +++ b/src/pumitally/PumiTallyImpl.cpp @@ -29,7 +29,8 @@ void TallyTimes::PrintTimes() const { } PumiTallyImpl::PumiTallyImpl(const std::string &mesh_filename, - const Omega_h::LO num_ptcls, int argc, char **argv, const SourceDistribution source_dist) + const Omega_h::LO num_ptcls, int argc, char **argv, + const SourceDistribution source_dist) : num_particles(num_ptcls) { oh_mesh_filename = mesh_filename; @@ -433,13 +434,14 @@ void apply_reflection_boundary_condition( // Material (%3d -> %3d)\n", // pid, elem_ids[pid], next_elems[pid], // class_ids[elem_ids[pid]], material_ids[pid]); - //printf("P %d in element %d hit lastExit %d xFace %d \n", pid, + // printf("P %d in element %d hit lastExit %d xFace %d \n", pid, // elem_ids[pid], lastExit[pid], xFace[pid]); - //xFace[pid] = lastExit[pid]; - Omega_h::LO hit_face = (lastExit[pid] == -1) ? xFace[pid] : lastExit[pid]; + // xFace[pid] = lastExit[pid]; + Omega_h::LO hit_face = + (lastExit[pid] == -1) ? xFace[pid] : lastExit[pid]; xFace[pid] = hit_face; lastExit[pid] = hit_face; - ptcl_done[pid] = 1; // stop the particle after reflection + ptcl_done[pid] = 1; // stop the particle after reflection next_elems[pid] = elem_ids[pid]; // reflects back to the same element OMEGA_H_CHECK_PRINTF(hit_face != -1, "Error: xFace[%d] is -1 but " @@ -456,8 +458,8 @@ void apply_reflection_boundary_condition( particle_destination(pid, 2) - particle_origin(pid, 2)}; // reflect the particle direction Omega_h::Vector<3> reflected_vector = - incident_vector - - 2.0 * Omega_h::inner_product(incident_vector, normal) * normal; + incident_vector - + 2.0 * Omega_h::inner_product(incident_vector, normal) * normal; // change the particle's position // particle reaches the boundary @@ -466,11 +468,11 @@ void apply_reflection_boundary_condition( particle_origin(pid, 2) = inter_points[pid * 3 + 2]; particle_destination(pid, 0) = - particle_origin(pid, 0) + reflected_vector[0]; + particle_origin(pid, 0) + reflected_vector[0]; particle_destination(pid, 1) = - particle_origin(pid, 1) + reflected_vector[1]; + particle_origin(pid, 1) + reflected_vector[1]; particle_destination(pid, 2) = - particle_origin(pid, 2) + reflected_vector[2]; + particle_origin(pid, 2) + reflected_vector[2]; } } }; @@ -525,11 +527,10 @@ void distributeParticlesBasesOnVolume(Omega_h::Mesh &mesh, o::parallel_for(std::abs(extra_particles), add_or_remove_particles); } - OMEGA_H_DEVICE o::Vector<3> -barycentric2real(const o::Few, 4>& tet_verts, -const o::Vector<4>& bary) { - o::Vector<3> real_coords {0,0,0}; +barycentric2real(const o::Few, 4> &tet_verts, + const o::Vector<4> &bary) { + o::Vector<3> real_coords{0, 0, 0}; for (int i = 0; i < 4; ++i) { real_coords += bary[i] * tet_verts[i]; } @@ -575,17 +576,17 @@ void initialize_uniform_source(Omega_h::Mesh &mesh, for (Omega_h::LO pid = pid_start; pid < pid_end; ++pid) { auto gen = random_pool.get_state(); o::Real r1 = gen.drand(0.0, 1.0); - o::Real r2 = gen.drand(0.0, 1.0); - o::Real r3 = gen.drand(0.0, 1.0); + o::Real r2 = gen.drand(0.0, 1.0); + o::Real r3 = gen.drand(0.0, 1.0); - r1 = Kokkos::pow(r1, 1.0 / 3.0); - r2 = Kokkos::sqrt(r2); - o::Real a = 1.0 - r1; - o::Real b = r1 * (1.0 - r2); - o::Real c = r1 * r2 * (1.0 - r3); - o::Real d = r1 * r2 * r3; + r1 = Kokkos::pow(r1, 1.0 / 3.0); + r2 = Kokkos::sqrt(r2); + o::Real a = 1.0 - r1; + o::Real b = r1 * (1.0 - r2); + o::Real c = r1 * r2 * (1.0 - r3); + o::Real d = r1 * r2 * r3; - o::Vector<4> random_bcc{a, b, c, d}; + o::Vector<4> random_bcc{a, b, c, d}; random_pool.free_state(gen); @@ -602,7 +603,6 @@ void initialize_uniform_source(Omega_h::Mesh &mesh, o::parallel_for(mesh.nelems(), set_initial_positions); } - void ApplyVacuumBC(const Omega_h::Mesh &mesh, PPPS *ptcls, const Omega_h::Write &elem_ids, const Omega_h::Write &next_elems, diff --git a/src/pumitally/PumiTallyImpl.h b/src/pumitally/PumiTallyImpl.h index b38f85a..eee4920 100644 --- a/src/pumitally/PumiTallyImpl.h +++ b/src/pumitally/PumiTallyImpl.h @@ -28,7 +28,7 @@ struct TallyTimes { enum class SourceDistribution { UNIFORM, // Source uniformly distributed across the mesh - EQUAL, // Source at centroids of each element + EQUAL, // Source at centroids of each element ZERO // in the zeroth element centroid }; @@ -198,7 +198,8 @@ struct PumiTallyImpl { TallyTimes tally_times; //!< Struct to hold times for different operations PumiTallyImpl(const std::string &mesh_filename, Omega_h::LO num_ptcls, - int argc, char **argv, SourceDistribution source_dist = SourceDistribution::ZERO); + int argc, char **argv, + SourceDistribution source_dist = SourceDistribution::ZERO); ~PumiTallyImpl() = default; diff --git a/test/test_degas2_physics.cpp b/test/test_degas2_physics.cpp index 0672448..640a6fd 100644 --- a/test/test_degas2_physics.cpp +++ b/test/test_degas2_physics.cpp @@ -7,16 +7,16 @@ #include "DG2Physics.h" #include +#include #include #include -#include TEST_CASE("Test Degas2 Physics Functions") { Kokkos::initialize(); { int numParticles = 1000; - double energy = 3.0; + double energy = 3.0; Kokkos::View sigma_t_; // mat, T, g Kokkos::View sigma_a_; // mat, T, g @@ -38,8 +38,8 @@ TEST_CASE("Test Degas2 Physics Functions") { particles(i).direction[2] = 0; particles(i).energy_group = energy; particles(i).weight = 1.0; - particles(i).particle_index = i; - particles(i).alpha = 1.0; + particles(i).particle_index = i; + particles(i).alpha = 1.0; fields(i).electron_density = 1.0e13; fields(i).ion_density = 1.0e13; @@ -48,101 +48,89 @@ TEST_CASE("Test Degas2 Physics Functions") { // and others like this }); - // Set the particle energy view - Kokkos::parallel_for( - "Set energies", numParticles, KOKKOS_LAMBDA(int i) { - physics.set_energy(particles(i), energy); - }); + // Set the particle energy view + Kokkos::parallel_for( + "Set energies", numParticles, + KOKKOS_LAMBDA(int i) { physics.set_energy(particles(i), energy); }); // Call the function to be tested - Kokkos::parallel_for( + Kokkos::parallel_for( "run physics", numParticles, KOKKOS_LAMBDA(int i) { physics.sample_collision_distance(particles(i), fields(i)); - physics.collide_particle(particles(i), fields(i)); + physics.collide_particle(particles(i), fields(i)); }); - auto output = create_mirror_view(particles); - Kokkos::deep_copy(output, particles); - std::ofstream outfile("Log.txt"); - - //These 4 functions should return the commented value for SEED=12345 - - double l {0}; - for (int i=0; i < numParticles; ++i) { - l += output(i).position[0]; - } - l /= numParticles; - outfile << "Average Distance (cm): " << l << std::endl; //2.41 - - double varl {0}; - for (int i=0; i < numParticles; ++i) { - varl += (output(i).position[0] - l)*(output(i).position[0] - l); - } - varl /= (numParticles - 1); - double sdl {sqrt(varl)}; - outfile << "Standard Deviation of Distance (cm): " << sdl << std::endl; //2.38 - - double ux {0}; - for (int i=0; i < numParticles; ++i) { - ux += output(i).direction[0]; - } - ux /= numParticles; - outfile << "Mean x Direction: " << ux << std::endl; //0.0159 - - double varux {0.0}; - for (int i=0; i < numParticles; ++i) { - varux += (output(i).direction[0] - ux)*(output(i).direction[0] - ux); - } - varux /= (numParticles - 1); - double sdux {sqrt(varux)}; - - - outfile << "Standard Deviation of Mean x Direction: " << sdux << std::endl; //0.579 - - - outfile << std::left - << std::setw(10) << "Particle #" - << std::setw(16) << "Velocity(m/s)" - << std::setw(10) << "Weight" - << std::setw(10) << "X(cm)" - << std::setw(10) << "Y(cm)" - << std::setw(10) << "Z(cm)" - << std::setw(10) << "X_Dir" - << std::setw(10) << "Y_Dir" - << std::setw(10) << "Z_Dir" - << '\n'; - - for (int i = 0; i < numParticles; ++i) { - outfile << std::left - << std::setw(10) << i - << std::setw(16) << (1.0 / output(i).alpha) - << std::setw(10) << output(i).weight - << std::setw(10) << output(i).position[0] - << std::setw(10) << output(i).position[1] - << std::setw(10) << output(i).position[2] - << std::setw(10) << output(i).direction[0] - << std::setw(10) << output(i).direction[1] - << std::setw(10) << output(i).direction[2] - << '\n'; - } - - // print the file - std::ifstream infile("Log.txt"); - std::string line; - while (std::getline(infile, line)) { - std::cout << line << '\n'; - } - outfile.close(); - - // Fixme: Where did this tolerance come from. Looks like it's - // not enough for 1000 particles. - // Tolerance is 0.0015 for l, with 95% confidence interval for mean - // with 1000 samples. But we should use something like 2-3 standard deviations - // with more than 99% confidence. - REQUIRE_THAT(l, Catch::Matchers::WithinAbs(0.0240,0.0015)); - REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(0.0240,0.0015)); - REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0,.03)); - REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577,.03)); + auto output = create_mirror_view(particles); + Kokkos::deep_copy(output, particles); + std::ofstream outfile("Log.txt"); + + // These 4 functions should return the commented value for SEED=12345 + + double l{0}; + for (int i = 0; i < numParticles; ++i) { + l += output(i).position[0]; + } + l /= numParticles; + outfile << "Average Distance (cm): " << l << std::endl; // 2.41 + + double varl{0}; + for (int i = 0; i < numParticles; ++i) { + varl += (output(i).position[0] - l) * (output(i).position[0] - l); + } + varl /= (numParticles - 1); + double sdl{sqrt(varl)}; + outfile << "Standard Deviation of Distance (cm): " << sdl + << std::endl; // 2.38 + + double ux{0}; + for (int i = 0; i < numParticles; ++i) { + ux += output(i).direction[0]; + } + ux /= numParticles; + outfile << "Mean x Direction: " << ux << std::endl; // 0.0159 + + double varux{0.0}; + for (int i = 0; i < numParticles; ++i) { + varux += (output(i).direction[0] - ux) * (output(i).direction[0] - ux); + } + varux /= (numParticles - 1); + double sdux{sqrt(varux)}; + + outfile << "Standard Deviation of Mean x Direction: " << sdux + << std::endl; // 0.579 + + outfile << std::left << std::setw(10) << "Particle #" << std::setw(16) + << "Velocity(m/s)" << std::setw(10) << "Weight" << std::setw(10) + << "X(cm)" << std::setw(10) << "Y(cm)" << std::setw(10) << "Z(cm)" + << std::setw(10) << "X_Dir" << std::setw(10) << "Y_Dir" + << std::setw(10) << "Z_Dir" << '\n'; + + for (int i = 0; i < numParticles; ++i) { + outfile << std::left << std::setw(10) << i << std::setw(16) + << (1.0 / output(i).alpha) << std::setw(10) << output(i).weight + << std::setw(10) << output(i).position[0] << std::setw(10) + << output(i).position[1] << std::setw(10) << output(i).position[2] + << std::setw(10) << output(i).direction[0] << std::setw(10) + << output(i).direction[1] << std::setw(10) + << output(i).direction[2] << '\n'; + } + + // print the file + std::ifstream infile("Log.txt"); + std::string line; + while (std::getline(infile, line)) { + std::cout << line << '\n'; + } + outfile.close(); + + // Fixme: Where did this tolerance come from. Looks like it's + // not enough for 1000 particles. + // Tolerance is 0.0015 for l, with 95% confidence interval for mean + // with 1000 samples. But we should use something like 2-3 standard + // deviations with more than 99% confidence. + REQUIRE_THAT(l, Catch::Matchers::WithinAbs(0.0240, 0.0015)); + REQUIRE_THAT(sdl, Catch::Matchers::WithinAbs(0.0240, 0.0015)); + REQUIRE_THAT(ux, Catch::Matchers::WithinAbs(0.0, .03)); + REQUIRE_THAT(sdux, Catch::Matchers::WithinAbs(0.577, .03)); } Kokkos::finalize(); - } From 11e8ecae0d9c64e5e4910ba9bea123bfbbc156f5 Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Thu, 30 Apr 2026 12:42:04 -0700 Subject: [PATCH 135/139] added dg2 physics format git-blame ignore --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 0da25a4..ca25877 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,2 +1,3 @@ c765e51344ad4a458d042ceefdf19fd61edf79ab # clang-format 59fe44fd6d08218b8aeae7612bdd5a71d0289ce5 # format /test/test_hostwrite_initialization.cpp +570b3252bcb6e080bde4113cc9d7ed2aa5796fd6 # format degas2 physics files From 3fe481f9d7055e6437048b034ee29d5efea38c75 Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Thu, 30 Apr 2026 13:30:05 -0700 Subject: [PATCH 136/139] moved get centroid method to impl header other methods may use it later. --- src/physics/degas2/degas2-main.cpp | 33 ++---------------------------- src/pumitally/PumiTallyImpl.cpp | 27 ++++++++++++++++++++++++ src/pumitally/PumiTallyImpl.h | 2 ++ 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index bfa77b4..ddc8bff 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -24,9 +24,7 @@ void read_input_parameters(int argc, char *const *argv, InputParameters ¶ms); void print_initial_info(const std::string &mesh_name, int num_particles) noexcept; -// computes centroids of each element in the mesh and stores them in a tag -void get_centroids(Omega_h::Mesh &mesh, - Omega_h::Write centroids); + void sample_initial_particle_energy_direction( Kokkos::View energy_array, Kokkos::View direction, Kokkos::View alpha); @@ -77,8 +75,7 @@ int main(int argc, char *argv[]) { input_params.mesh_name, input_params.num_particles, argc, argv, input_params.source_distribution); auto &mesh = pumi_tally.full_mesh; - Omega_h::Write centroids(mesh.nelems() * 3); - get_centroids(mesh, centroids); + const auto centroids = pumitally::GetCentroids(mesh); Fields fields; get_field_values(Omega_h::Reals(centroids), fields); set_field_values_to_mesh(mesh, fields); @@ -326,32 +323,6 @@ void set_field_values_to_mesh(Omega_h::Mesh &mesh, const Fields &fields) { bulk_flow_velocity); } -void get_centroids(Omega_h::Mesh &mesh, - Omega_h::Write centroids) { - OMEGA_H_CHECK_PRINTF( - centroids.size() == mesh.nelems() * 3, - "Centroids size (%d) must be equal to number of elements * 3 (%d)\n", - centroids.size(), mesh.nelems() * 3); - const auto coords = mesh.coords(); - const auto nelems = mesh.nelems(); - const auto e2v = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; - - // FIXME: Hardcoded for 3D tets - Omega_h::parallel_for( - "calculate centroids", nelems, OMEGA_H_LAMBDA(int e) { - auto nodes = o::gather_verts<4>(e2v, e); - o::Few, 4> elem_coords = - o::gather_vectors<4, 3>(coords, nodes); - o::Vector<3> centroid = o::average(elem_coords); - - centroids[e * 3 + 0] = centroid[0]; - centroids[e * 3 + 1] = centroid[1]; - centroids[e * 3 + 2] = centroid[2]; - }); - - mesh.add_tag(Omega_h::REGION, "centroid", 3, centroids); -} - // TODO: Implement the function to retrieve field values based on centroids // NOTE: This requires the python script and the plasma source mesh and data // to be in the folder with this cpp file. The python script may need to be diff --git a/src/pumitally/PumiTallyImpl.cpp b/src/pumitally/PumiTallyImpl.cpp index fc0bdc0..82ca6ee 100644 --- a/src/pumitally/PumiTallyImpl.cpp +++ b/src/pumitally/PumiTallyImpl.cpp @@ -19,6 +19,33 @@ std::unique_ptr CreateParticleDS(const Omega_h::Mesh &mesh, void InitializeParticlesInElement0(Omega_h::Mesh &mesh, pumitally::PPPS *ptcls); +Omega_h::Reals GetCentroids(Omega_h::Mesh &mesh, const bool add_tag) { + const auto coords = mesh.coords(); + const auto nelems = mesh.nelems(); + const auto e2v = mesh.ask_down(Omega_h::REGION, Omega_h::VERT).ab2b; + + const Omega_h::Write centroids(nelems * 3, 0.0, "centroids"); + + // FIXME: Hardcoded for 3D tets + Omega_h::parallel_for( + "calculate centroids", nelems, OMEGA_H_LAMBDA(int e) { + const auto nodes = o::gather_verts<4>(e2v, e); + o::Few, 4> elem_coords = + o::gather_vectors<4, 3>(coords, nodes); + o::Vector<3> centroid = o::average(elem_coords); + + centroids[e * 3 + 0] = centroid[0]; + centroids[e * 3 + 1] = centroid[1]; + centroids[e * 3 + 2] = centroid[2]; + }); + + if (add_tag) { + mesh.add_tag(Omega_h::REGION, "centroid", 3, centroids); + } + + return centroids; +} + void TallyTimes::PrintTimes() const { printf("\n"); printf("[TIME] Initialization time : %f seconds\n", initialization_time); diff --git a/src/pumitally/PumiTallyImpl.h b/src/pumitally/PumiTallyImpl.h index eee4920..59c8ba0 100644 --- a/src/pumitally/PumiTallyImpl.h +++ b/src/pumitally/PumiTallyImpl.h @@ -12,6 +12,8 @@ namespace pumitally { +Omega_h::Reals GetCentroids(Omega_h::Mesh &mesh, bool add_tag = true); + /** * Data structure to hold the timing information for different sections */ From de78f4c2791f17f18bdfd008d1ae6a55f9a4857e Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Thu, 30 Apr 2026 14:23:14 -0700 Subject: [PATCH 137/139] Refactor degas2-main.cpp - Updated function names to follow CamelCase. - Changed parameter types in function signatures to use references for efficiency. Although most of them are called only once. - Refactored InputParameter struct and make the reader a constructor for better structure. - Added some todos for the next commit. --- src/physics/degas2/degas2-main.cpp | 175 +++++++++++++++-------------- 1 file changed, 90 insertions(+), 85 deletions(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index ddc8bff..a25cf6e 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -14,20 +14,19 @@ #include "PumiTallyImpl.h" struct InputParameters { + InputParameters(int argc, char *argv[]); std::string mesh_name; int num_particles = 0; int max_iterations = 1000; pumitally::SourceDistribution source_distribution = pumitally::SourceDistribution::EQUAL; }; -void read_input_parameters(int argc, char *const *argv, - InputParameters ¶ms); -void print_initial_info(const std::string &mesh_name, - int num_particles) noexcept; -void sample_initial_particle_energy_direction( - Kokkos::View energy_array, Kokkos::View direction, - Kokkos::View alpha); +void PrintInitialInfo(const std::string &mesh_name, int num_particles) noexcept; + +void SampleInitialParticleState(const Kokkos::View &energy_array, + const Kokkos::View &direction, + const Kokkos::View &alpha); template void vector2write(const std::vector &vec, Omega_h::Write &write_vec) { @@ -38,6 +37,8 @@ void vector2write(const std::vector &vec, Omega_h::Write &write_vec) { write_vec = Omega_h::Write(host_write_vec); } +// todo: make it a set of reads, not writes +// make the read field values and set fields to mesh members struct Fields { std::vector electron_temperature; std::vector ion_temperature; @@ -45,19 +46,19 @@ struct Fields { std::vector ion_density; std::vector bulk_flow_velocity; }; -void get_field_values(Omega_h::Reals centroids, Fields &fields); +void get_field_values(const Omega_h::Reals ¢roids, Fields &fields); void set_field_values_to_mesh(Omega_h::Mesh &mesh, const Fields &fields); // void set_source_particles(Omega_h::Mesh &mesh, // SourceDistribution source_distribution, // Omega_h::Write particle_positions); -void transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, +void Transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, const Omega_h::Read &electron_density, - const Omega_h::Read ion_density, - const Omega_h::Read electron_temperature, - const Omega_h::Read ion_temperature, - const Omega_h::Read bulk_flow_velocity, - const Kokkos::View initial_direction, - const Kokkos::View initial_alpha, int max_iterations); + const Omega_h::Read &ion_density, + const Omega_h::Read &electron_temperature, + const Omega_h::Read &ion_temperature, + const Omega_h::Read &bulk_flow_velocity, + const Kokkos::View &initial_direction, + const Kokkos::View &initial_alpha, int max_iterations); // ***************************************************************************// // *************************** Main Function ******************************** // @@ -66,9 +67,8 @@ int main(int argc, char *argv[]) { Kokkos::initialize(argc, argv); { // Read input parameters - InputParameters input_params; - read_input_parameters(argc, argv, input_params); - print_initial_info(input_params.mesh_name, input_params.num_particles); + InputParameters input_params(argc, argv); + PrintInitialInfo(input_params.mesh_name, input_params.num_particles); // Initialize PUMI-Tally and Read Fields auto pumi_tally = pumitally::PumiTallyImpl( @@ -81,20 +81,19 @@ int main(int argc, char *argv[]) { set_field_values_to_mesh(mesh, fields); pumi_tally.is_pumipic_initialized = true; - Kokkos::View sigma_t_; // mat, T, g - Kokkos::View sigma_a_; // mat, T, g - Kokkos::View scattering_matrix_; // mat, T, g, g - Kokkos::View sigma_s_; // mat, T, g - DG2CrossSection crossSection(sigma_t_, sigma_a_, scattering_matrix_, - sigma_s_); - DG2Physics physics(crossSection, input_params.num_particles); + Kokkos::View sigma_t; // mat, T, g + Kokkos::View sigma_a; // mat, T, g + Kokkos::View scattering_matrix; // mat, T, g, g + Kokkos::View sigma_s; // mat, T, g + DG2CrossSection cross_section(sigma_t, sigma_a, scattering_matrix, sigma_s); + DG2Physics physics(cross_section, input_params.num_particles); Kokkos::View initial_direction("Initial Direction", input_params.num_particles * 3); Kokkos::View initial_alpha("Initial Alpha", input_params.num_particles); - sample_initial_particle_energy_direction(physics.particle_energy, - initial_direction, initial_alpha); + SampleInitialParticleState(physics.particle_energy, initial_direction, + initial_alpha); // Move particles auto electron_density = @@ -108,7 +107,7 @@ int main(int argc, char *argv[]) { auto bulk_flow_velocity = mesh.get_array(Omega_h::REGION, "bulk_flow_velocity"); - transport(pumi_tally, physics, electron_density, ion_density, + Transport(pumi_tally, physics, electron_density, ion_density, electron_temperature, ion_temperature, bulk_flow_velocity, initial_direction, initial_alpha, input_params.max_iterations); @@ -127,14 +126,15 @@ int main(int argc, char *argv[]) { // **************************************************************************************************************** // // -void transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, +void Transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, const Omega_h::Read &electron_density, - const Omega_h::Read ion_density, - const Omega_h::Read electron_temperature, - const Omega_h::Read ion_temperature, - const Omega_h::Read bulk_flow_velocity, - const Kokkos::View initial_direction, - const Kokkos::View initial_alpha, int max_iterations) { + const Omega_h::Read &ion_density, + const Omega_h::Read &electron_temperature, + const Omega_h::Read &ion_temperature, + const Omega_h::Read &bulk_flow_velocity, + const Kokkos::View &initial_direction, + const Kokkos::View &initial_alpha, + int max_iterations) { for (int iter = 0; iter < max_iterations; ++iter) { auto particle_dest = pumi_tally.pumipic_ptcls->get<1>(); @@ -327,7 +327,7 @@ void set_field_values_to_mesh(Omega_h::Mesh &mesh, const Fields &fields) { // NOTE: This requires the python script and the plasma source mesh and data // to be in the folder with this cpp file. The python script may need to be // updated to read in different plasma source data. -void get_field_values(Omega_h::Reals centroids, Fields &fields) { +void get_field_values(const Omega_h::Reals ¢roids, Fields &fields) { auto centroids_h = Omega_h::HostRead(centroids); // This function should be implemented to retrieve the field values @@ -342,44 +342,43 @@ void get_field_values(Omega_h::Reals centroids, Fields &fields) { fields.bulk_flow_velocity.resize(3 * num_elements, 0.0); // TODO: Add your code here to retrieve the actual field values - // Write centroid coords to CSV to be read by python script + // Write centroid coords to CSV to be read by Python script std::ofstream coords("centroid_coords.csv"); for (int j = 0; j < (centroids_h.size() / 3); j++) { coords << j << "," << centroids_h[j * 3 + 0] << "," - << centroids_h[j * 3 + 1] << "," << centroids_h[j * 3 + 2] - << std::endl; + << centroids_h[j * 3 + 1] << "," << centroids_h[j * 3 + 2] << '\n'; } coords.close(); - // Run the python script + // Run the Python script // system("python Mesh_Map.py"); // Read in the field values std::ifstream fieldvals{"vals.csv"}; int i = 0; - std::string IDString; - std::string xxString; - std::string yyString; - std::string zzString; + std::string id_string; + std::string xx_string; + std::string yy_string; + std::string zz_string; - while (std::getline(fieldvals, IDString, ',')) { + while (std::getline(fieldvals, id_string, ',')) { - std::getline(fieldvals, xxString, ','); - std::getline(fieldvals, yyString, ','); - std::getline(fieldvals, zzString, '\n'); + std::getline(fieldvals, xx_string, ','); + std::getline(fieldvals, yy_string, ','); + std::getline(fieldvals, zz_string, '\n'); // Asign field values - fields.ion_density[i] = std::stod(xxString); - fields.electron_density[i] = std::stod(xxString); - fields.electron_temperature[i] = std::stod(yyString); - fields.ion_temperature[i] = std::stod(zzString); + fields.ion_density[i] = std::stod(xx_string); + fields.electron_density[i] = std::stod(xx_string); + fields.electron_temperature[i] = std::stod(yy_string); + fields.ion_temperature[i] = std::stod(zz_string); i++; } } -void print_initial_info(const std::string &mesh_name, - int num_particles) noexcept { +void PrintInitialInfo(const std::string &mesh_name, + int num_particles) noexcept { printf("\n===================> Accelerated Degas2 <======================\n"); std::string mesh_particle_info = "\tMesh: " + mesh_name + @@ -388,50 +387,53 @@ void print_initial_info(const std::string &mesh_name, printf("\n===============================================================\n"); } -void read_input_parameters(int argc, char *const *argv, - InputParameters ¶ms) { +InputParameters::InputParameters(const int argc, char *argv[]) { if (argc != 5) { throw std::runtime_error( "Usage: " + std::string(argv[0]) + " "); } - params.mesh_name = argv[1]; - params.num_particles = std::stoi(argv[2]); - if (params.num_particles <= 0) { + mesh_name = argv[1]; + num_particles = std::stoi(argv[2]); + if (num_particles <= 0) { throw std::runtime_error("Number of particles must be a positive integer."); } - params.max_iterations = std::stoi(argv[3]); + max_iterations = std::stoi(argv[3]); std::string source_dist_str = argv[4]; // FIXME: hardcoded lower case if (source_dist_str == "uniform") { - params.source_distribution = pumitally::SourceDistribution::UNIFORM; + source_distribution = pumitally::SourceDistribution::UNIFORM; } else if (source_dist_str == "equal") { - params.source_distribution = pumitally::SourceDistribution::EQUAL; + source_distribution = pumitally::SourceDistribution::EQUAL; } else { throw std::runtime_error( "Invalid source distribution. Use 'uniform' or 'equal'."); } } -void sample_initial_particle_energy_direction( - Kokkos::View energy_array, Kokkos::View direction, - Kokkos::View alpha) { - random_pool_t randomPool(0); - auto sample_energy = OMEGA_H_LAMBDA(int i) { +// todo: split this function to sample uniform direction and sample energy +// and move to impl class +void SampleInitialParticleState(const Kokkos::View &energy_array, + Kokkos::View &direction, + const Kokkos::View &alpha) { + const random_pool_t random_pool(SEED); + + auto sample_energy = OMEGA_H_LAMBDA(const int i) { // Basically, feed this 4 uniformly generated random numbers, x1, x2, y1, // y2, on the interval (0,1) and it // will give you a velocity vector (vx,vy,vz), a unit direction vector // (directionx, directiony, directionz), the alpha value used in the tally, // and the energy of the particle that was sampled. All from a gas at // temperature temp. - double mp{938.27e6 / (3e10 * 3e10)}; // eV/c^2 = eV*s^2/cm^2. Necessary + constexpr double kMp{938.27e6 / + (3e10 * 3e10)}; // eV/c^2 = eV*s^2/cm^2. Necessary // constant for the distribution // This is what we set as the source temperature. This could in principle be // a parameter but for now can be hard coded in as 3 eV or so and I can // always tweak it. - double temp = 3; // eV. + constexpr double kSourceTemp = 3; // eV. // This uses the Box-Muller method of sampling a Gaussian, which generates // two independent normally distributed values from two independent @@ -440,21 +442,24 @@ void sample_initial_particle_energy_direction( // 4 independent normally distributed values, but we only need three. This // generates a velocity vector (vx,vy,vz) sampled from an ideal gas at // temperature temp. - auto generator = randomPool.get_state(); - auto x1 = generator.drand(0.0, 1.0); - auto x2 = generator.drand(0.0, 1.0); - auto y1 = generator.drand(0.0, 1.0); - auto y2 = generator.drand(0.0, 1.0); - randomPool.free_state(generator); - auto vx = Kokkos::sqrt(temp / mp) * Kokkos::sqrt(-2 * Kokkos::log(x1)) * - Kokkos::cos(2 * M_PI * x2); - auto vy = Kokkos::sqrt(temp / mp) * Kokkos::sqrt(-2 * Kokkos::log(x1)) * - Kokkos::sin(2 * M_PI * x2); - auto vz = Kokkos::sqrt(temp / mp) * Kokkos::sqrt(-2 * Kokkos::log(y1)) * - Kokkos::sin(2 * M_PI * y2); + auto generator = random_pool.get_state(); + const double x1 = generator.drand(0.0, 1.0); + const double x2 = generator.drand(0.0, 1.0); + const double y1 = generator.drand(0.0, 1.0); + const double y2 = generator.drand(0.0, 1.0); + random_pool.free_state(generator); + const double vx = Kokkos::sqrt(kSourceTemp / kMp) * + Kokkos::sqrt(-2 * Kokkos::log(x1)) * + Kokkos::cos(2 * M_PI * x2); + const double vy = Kokkos::sqrt(kSourceTemp / kMp) * + Kokkos::sqrt(-2 * Kokkos::log(x1)) * + Kokkos::sin(2 * M_PI * x2); + const double vz = Kokkos::sqrt(kSourceTemp / kMp) * + Kokkos::sqrt(-2 * Kokkos::log(y1)) * + Kokkos::sin(2 * M_PI * y2); // Compute the magnitude of the velocity vector - auto mag_v = Kokkos::sqrt(vx * vx + vy * vy + vz * vz); + const double mag_v = Kokkos::sqrt(vx * vx + vy * vy + vz * vz); // Compute the alpha factor multiplied in when computing the tally alpha(i) = 1 / mag_v; @@ -467,9 +472,9 @@ void sample_initial_particle_energy_direction( // Compute the particle energy. This you will want to save probably to pass // to my initialization (set_energy) code so that I have the energy for my // computations. - double particle_energy = 0.5 * mp * mag_v * mag_v; + const double particle_energy = 0.5 * kMp * mag_v * mag_v; energy_array(i) = particle_energy; }; - Kokkos::parallel_for("sample_initial_particle_energy_direction", - energy_array.size(), sample_energy); + Kokkos::parallel_for("SampleInitialParticleState", energy_array.size(), + sample_energy); } From 9c39acfe1990e54df400dab308d7a05465feaf1d Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Fri, 1 May 2026 10:52:05 -0700 Subject: [PATCH 138/139] fix function signature --- src/physics/degas2/degas2-main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index a25cf6e..707cd1d 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -415,7 +415,7 @@ InputParameters::InputParameters(const int argc, char *argv[]) { // todo: split this function to sample uniform direction and sample energy // and move to impl class void SampleInitialParticleState(const Kokkos::View &energy_array, - Kokkos::View &direction, + const Kokkos::View &direction, const Kokkos::View &alpha) { const random_pool_t random_pool(SEED); From 99fb85c7ebafdfd0b23b72bedd6d853b13e01463 Mon Sep 17 00:00:00 2001 From: Md Fuad Hasibul Hasan Date: Wed, 10 Jun 2026 06:19:14 -0700 Subject: [PATCH 139/139] refactor fields struct replace reading to vector with hostwrite, writetag function --- src/physics/degas2/degas2-main.cpp | 115 +++++++++++------------------ 1 file changed, 43 insertions(+), 72 deletions(-) diff --git a/src/physics/degas2/degas2-main.cpp b/src/physics/degas2/degas2-main.cpp index 707cd1d..9119861 100644 --- a/src/physics/degas2/degas2-main.cpp +++ b/src/physics/degas2/degas2-main.cpp @@ -28,29 +28,22 @@ void SampleInitialParticleState(const Kokkos::View &energy_array, const Kokkos::View &direction, const Kokkos::View &alpha); -template -void vector2write(const std::vector &vec, Omega_h::Write &write_vec) { - Omega_h::HostWrite host_write_vec(vec.size()); - for (size_t i = 0; i < vec.size(); ++i) { - host_write_vec[i] = vec[i]; - } - write_vec = Omega_h::Write(host_write_vec); -} - -// todo: make it a set of reads, not writes -// make the read field values and set fields to mesh members struct Fields { - std::vector electron_temperature; - std::vector ion_temperature; - std::vector electron_density; - std::vector ion_density; - std::vector bulk_flow_velocity; + explicit Fields(const Omega_h::Reals ¢roids); + + /** + * Set the fields as mesh tag + * @param mesh Mesh to tag + */ + void SetAsTag(Omega_h::Mesh &mesh) const; + + Omega_h::Reals electron_temperature; + Omega_h::Reals ion_temperature; + Omega_h::Reals electron_density; + Omega_h::Reals ion_density; + Omega_h::Reals bulk_flow_velocity; }; -void get_field_values(const Omega_h::Reals ¢roids, Fields &fields); -void set_field_values_to_mesh(Omega_h::Mesh &mesh, const Fields &fields); -// void set_source_particles(Omega_h::Mesh &mesh, -// SourceDistribution source_distribution, -// Omega_h::Write particle_positions); + void Transport(pumitally::PumiTallyImpl &pumi_tally, DG2Physics &physics, const Omega_h::Read &electron_density, const Omega_h::Read &ion_density, @@ -76,9 +69,8 @@ int main(int argc, char *argv[]) { input_params.source_distribution); auto &mesh = pumi_tally.full_mesh; const auto centroids = pumitally::GetCentroids(mesh); - Fields fields; - get_field_values(Omega_h::Reals(centroids), fields); - set_field_values_to_mesh(mesh, fields); + Fields fields(centroids); + fields.SetAsTag(mesh); pumi_tally.is_pumipic_initialized = true; Kokkos::View sigma_t; // mat, T, g @@ -275,41 +267,27 @@ void set_source_particles(Omega_h::Mesh &mesh, } */ -void set_field_values_to_mesh(Omega_h::Mesh &mesh, const Fields &fields) { - OMEGA_H_CHECK_PRINTF(fields.electron_temperature.size() == mesh.nelems(), +void Fields::SetAsTag(Omega_h::Mesh &mesh) const { + OMEGA_H_CHECK_PRINTF(electron_temperature.size() == mesh.nelems(), "Electron temperature size (%d) must be equal to number " "of elements (%d)\n", - fields.electron_temperature.size(), mesh.nelems()); + electron_temperature.size(), mesh.nelems()); OMEGA_H_CHECK_PRINTF( - fields.ion_temperature.size() == mesh.nelems(), + ion_temperature.size() == mesh.nelems(), "Ion temperature size (%d) must be equal to number of elements (%d)\n", - fields.ion_temperature.size(), mesh.nelems()); + ion_temperature.size(), mesh.nelems()); OMEGA_H_CHECK_PRINTF( - fields.electron_density.size() == mesh.nelems(), + electron_density.size() == mesh.nelems(), "Electron density size (%d) must be equal to number of elements (%d)\n", - fields.electron_density.size(), mesh.nelems()); + electron_density.size(), mesh.nelems()); OMEGA_H_CHECK_PRINTF( - fields.ion_density.size() == mesh.nelems(), + ion_density.size() == mesh.nelems(), "Ion density size (%d) must be equal to number of elements (%d)\n", - fields.ion_density.size(), mesh.nelems()); - OMEGA_H_CHECK_PRINTF(fields.bulk_flow_velocity.size() == mesh.nelems() * 3, + ion_density.size(), mesh.nelems()); + OMEGA_H_CHECK_PRINTF(bulk_flow_velocity.size() == mesh.nelems() * 3, "Bulk flow velocity size (%d) must be equal to number " "of elements * 3 (%d)\n", - fields.bulk_flow_velocity.size(), mesh.nelems() * 3); - - Omega_h::Write electron_temperature( - fields.electron_temperature.size()); - Omega_h::Write ion_temperature(fields.ion_temperature.size()); - Omega_h::Write electron_density( - fields.electron_density.size()); - Omega_h::Write ion_density(fields.ion_density.size()); - Omega_h::Write bulk_flow_velocity( - fields.bulk_flow_velocity.size()); - vector2write(fields.electron_temperature, electron_temperature); - vector2write(fields.ion_temperature, ion_temperature); - vector2write(fields.electron_density, electron_density); - vector2write(fields.ion_density, ion_density); - vector2write(fields.bulk_flow_velocity, bulk_flow_velocity); + bulk_flow_velocity.size(), mesh.nelems() * 3); // Assuming the mesh has a field for each of the properties mesh.add_tag(Omega_h::REGION, "electron_temperature", 1, @@ -327,31 +305,19 @@ void set_field_values_to_mesh(Omega_h::Mesh &mesh, const Fields &fields) { // NOTE: This requires the python script and the plasma source mesh and data // to be in the folder with this cpp file. The python script may need to be // updated to read in different plasma source data. -void get_field_values(const Omega_h::Reals ¢roids, Fields &fields) { +Fields::Fields(const Omega_h::Reals ¢roids) { auto centroids_h = Omega_h::HostRead(centroids); // This function should be implemented to retrieve the field values // based on the centroids provided (read centroids_h on CPU. // It contains centroids like e0_0, e0_1, e0_2, e1_0, e1_1, e1_2, ....) // For now, we will just fill the fields with dummy data. - size_t num_elements = centroids.size() / 3; // Assuming 3D - fields.electron_temperature.resize(num_elements, 1.0); - fields.ion_temperature.resize(num_elements, 1.0); - fields.electron_density.resize(num_elements, 1.0); - fields.ion_density.resize(num_elements, 1.0); - fields.bulk_flow_velocity.resize(3 * num_elements, 0.0); - - // TODO: Add your code here to retrieve the actual field values - // Write centroid coords to CSV to be read by Python script - std::ofstream coords("centroid_coords.csv"); - for (int j = 0; j < (centroids_h.size() / 3); j++) { - coords << j << "," << centroids_h[j * 3 + 0] << "," - << centroids_h[j * 3 + 1] << "," << centroids_h[j * 3 + 2] << '\n'; - } - coords.close(); - - // Run the Python script - // system("python Mesh_Map.py"); + const Omega_h::LO num_elements = centroids.size() / 3; // Assuming 3D + Omega_h::HostWrite host_electron_temperature(num_elements, "host_e_temp"); + Omega_h::HostWrite host_ion_temperature(num_elements, "host_i_temp"); + Omega_h::HostWrite host_electron_density(num_elements, "host_e_density"); + Omega_h::HostWrite host_ion_density(num_elements, "host_i_density"); + Omega_h::HostWrite host_bulk_flow_velocity(3 * num_elements, "host_bulk_flow_velocity"); // Read in the field values std::ifstream fieldvals{"vals.csv"}; @@ -368,13 +334,18 @@ void get_field_values(const Omega_h::Reals ¢roids, Fields &fields) { std::getline(fieldvals, zz_string, '\n'); // Asign field values - fields.ion_density[i] = std::stod(xx_string); - fields.electron_density[i] = std::stod(xx_string); - fields.electron_temperature[i] = std::stod(yy_string); - fields.ion_temperature[i] = std::stod(zz_string); + host_ion_density[i] = std::stod(xx_string); + host_electron_density[i] = std::stod(xx_string); + host_electron_temperature[i] = std::stod(yy_string); + host_ion_temperature[i] = std::stod(zz_string); i++; } + + ion_density = Omega_h::Reals(host_ion_density); + electron_density = Omega_h::Reals(host_electron_density); + electron_temperature = Omega_h::Reals(host_electron_temperature); + ion_temperature = Omega_h::Reals(host_ion_temperature); } void PrintInitialInfo(const std::string &mesh_name,