From 17341585d0049cfc351e5003d19fdf516a86adf7 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Sun, 12 Jul 2026 21:07:33 -0600 Subject: [PATCH 01/28] Initial commit. Checkpoint version 2->3. AB terms written in Chebyshev space. --- src/IO/Parallel_IO.F90 | 3 ++- src/Physics/Checkpointing.F90 | 26 ++++++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/IO/Parallel_IO.F90 b/src/IO/Parallel_IO.F90 index 5de1b793a..3fa52b464 100644 --- a/src/IO/Parallel_IO.F90 +++ b/src/IO/Parallel_IO.F90 @@ -180,7 +180,7 @@ Subroutine Initialize_IO_Buffer(self,grid_pars, nvals, mpi_tag, & averaging_weights, nrec, skip, & write_timestamp, averaging_axes, & spectral, mode, l_values, cache_spectral, & - spec_comp, lmax_in, full_cache) + spec_comp, nr_in, lmax_in, full_cache) Implicit None Class(io_buffer) :: self Integer, Intent(In) :: grid_pars(1:,1:) @@ -191,6 +191,7 @@ Subroutine Initialize_IO_Buffer(self,grid_pars, nvals, mpi_tag, & Logical, Intent(In), Optional :: full_cache, spec_comp Integer, Intent(In), Optional :: averaging_axes(3) Integer, Intent(In), Optional :: mode + Integer, Intent(In), Optional :: nr_in Integer, Intent(In), Optional :: lmax_in Integer :: adim(2), in_lmax Real*8, Allocatable :: avg_weights(:,:) diff --git a/src/Physics/Checkpointing.F90 b/src/Physics/Checkpointing.F90 index 7b127a5d0..4dbec84ae 100755 --- a/src/Physics/Checkpointing.F90 +++ b/src/Physics/Checkpointing.F90 @@ -36,10 +36,10 @@ Module Checkpointing ! Simple Checkpointing Module ! Uses MPI-IO to split writing of files amongst rank zero processes from each row Implicit None - Type(SphericalBuffer) :: chktmp, chktmp2, bctmp + Type(SphericalBuffer) :: chktmp, chktmp2, bctmp,abterms_cheby Integer, private :: numfields Integer, private :: check_err_off = 100 ! Checkpoint errors report in range 100-200. - Integer, private :: checkpoint_version = 2 + Integer, private :: checkpoint_version = 3 Integer,private :: checkpoint_tag = 425 Character*120 :: checkpoint_prefix ='nothing' Character*6 :: auto_fmt = '(i2.2)' ! Format code for quicksaves @@ -138,6 +138,10 @@ Subroutine Initialize_Checkpointing() Call checkpoint_buffer%Init(gpars, mpi_tag=checkpoint_tag, & spectral=.true., cache_spectral = .true., spec_comp = .true.) DeAllocate(gpars) + + ! Buffer to hold Adams-Bashforth terms in Chebyshev format + nfs(:) = numfields + Call abterms_cheby%init(field_count = nfs, config = 'p1a') End Subroutine Initialize_Checkpointing Subroutine Write_Checkpoint(abterms,iteration,dt,new_dt,elapsed_time, input_file) @@ -152,10 +156,20 @@ Subroutine Write_Checkpoint(abterms,iteration,dt,new_dt,elapsed_time, input_file endian_tag=314 Call chktmp%construct('p1a') chktmp%config = 'p1a' - !Copy the RHS into chtkmp - Call Get_All_RHS(chktmp%p1a) - chktmp%p1a(:,:,:,numfields+1:numfields*2) = abterms(:,:,:,1:numfields) - !Now we want to move from p1a to s2a (rlm space) + + !Copy the RHS (contains state variables at current timestep) into chtkmp + Call Get_All_RHS(chktmp%p1a) + + ! Next, convert the AB terms into chebyshev space and copy them into the buffer + Call abterms_cheby%construct('p1a') + abterms_cheby%config='p1a' + !chktmp%p1a(:,:,:,numfields+1:numfields*2) = abterms(:,:,:,1:numfields) + ! TRANSFORM ABTERMS CHEBY + Call gridcp%to_spectral(abterms,abterms_cheby%p1a) + chktmp%p1a(:,:,:,numfields+1:numfields*2) = abterms_cheby%p1a(:,:,:,1:numfields) + Call abterms_cheby%deconstruct('p1a') + + !Move checkpoint buffer from p1a to s2a (rlm space) Call chktmp%reform() If (ItIsTimeForAQuickSave) Then From 84ead8b3a8506dee7727bb45d4db7d5a37db47f9 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Mon, 13 Jul 2026 10:52:25 -0600 Subject: [PATCH 02/28] Interpolation from lower resolution now adjusted. --- src/Physics/Checkpointing.F90 | 66 ++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/src/Physics/Checkpointing.F90 b/src/Physics/Checkpointing.F90 index 4dbec84ae..f3435da9c 100755 --- a/src/Physics/Checkpointing.F90 +++ b/src/Physics/Checkpointing.F90 @@ -163,8 +163,7 @@ Subroutine Write_Checkpoint(abterms,iteration,dt,new_dt,elapsed_time, input_file ! Next, convert the AB terms into chebyshev space and copy them into the buffer Call abterms_cheby%construct('p1a') abterms_cheby%config='p1a' - !chktmp%p1a(:,:,:,numfields+1:numfields*2) = abterms(:,:,:,1:numfields) - ! TRANSFORM ABTERMS CHEBY + !chktmp%p1a(:,:,:,numfields+1:numfields*2) = abterms(:,:,:,1:numfields) ! earlier version -- physical space storage Call gridcp%to_spectral(abterms,abterms_cheby%p1a) chktmp%p1a(:,:,:,numfields+1:numfields*2) = abterms_cheby%p1a(:,:,:,1:numfields) Call abterms_cheby%deconstruct('p1a') @@ -577,6 +576,14 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Call chktmp%reform() ! move to p1b + If (version .ge. 2) Then + ! For version 2+, the AB terms are stored in Chebyshev space + Call abterms_cheby%construct('p1b') + abterms_cheby%config='p1b' + abterms_cheby%p1b(:,:,:,1:numfields)=chktmp%p1b(:,:,:,numfields+1:numfields*2) + Endif + + If (.not. legacy_format) Then ! Load the boundary values array @@ -627,44 +634,55 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Call stdout%print(' ') Endif - If (n_r_old .lt. n_r) Then + + + If ((n_r_old .lt. n_r) .and. (version .lt. 2) ) Then + + ! Prior to version 2, the AB terms were stored in radial-physical space rather + ! than in radial-Chebyshev space. + ! Before proceeding, we need to convert to Chebyshev space. + ! During the read-in process above, old fields were stored in + ! indices 1:n_r_old of first dimension of chktmp%p1b. - ! The fields are OK - they are already in chebyshev space - fields(:,:,:,1:numfields) = chktmp%p1b(:,:,:,1:numfields) - ! The AB terms are stored in physical space (in radius). - ! They need to be transformed, coefficients copied, and transformed back.. - ! First, we need to initialize the old chebyshev grid. + ! (1) Initialize the old Chebyshev grid. Allocate(radius_old(1:n_r_old)) - Call cheby_info%Init(radius_old,rmin,rmax) ! We assume that rmax and rmin do not change + Call cheby_info%Init(radius_old,rmin,rmax) ! rmax and rmin cannot change + + ! (2) Create some buffer space to store the old radial profiles and + ! the new Chebyshev coefficients. fcount(:,:) = numfields Call chktmp2%init(field_count = fcount, config = 'p1a') Call chktmp2%construct('p1a') chktmp2%p1a(:,:,:,:) = 0.0d0 - ! Allocate tempfield1, tempfield2 lb = lbound(chktmp%p1b,3) ub = ubound(chktmp%p1b,3) Allocate(tempfield1(1:n_r_old,1:2,lb:ub,1)) Allocate(tempfield2(1:n_r_old,1:2,lb:ub,1)) + ! (3) Convert to the new Chebyshev grid Do i = 1, numfields tempfield1(:,:,:,:) = 0.0d0 tempfield2(:,:,:,:) = 0.0d0 + ! Subsample smaller-grid fields from larger-grid buffer tempfield1(1:n_r_old,:,:,1) = chktmp%p1b(1:n_r_old,:,:,numfields+i) - call cheby_info%tospec4d(tempfield1,tempfield2) + ! Transform + Call cheby_info%tospec4d(tempfield1,tempfield2) + + ! Grab Chebyshev coefficients up to old n_max chktmp2%p1a(1:n_r_old,:,:,i) = tempfield2(1:n_r_old,:,:,1) Enddo DeAllocate(tempfield1,tempfield2) - Call chktmp2%construct('p1b') + !Call chktmp2%construct('p1b') !Normal transform(p1a,p1b) - Call gridcp%From_Spectral(chktmp2%p1a,chktmp2%p1b) + !Call gridcp%From_Spectral(chktmp2%p1a,chktmp2%p1b) - abterms(:,:,:,1:numfields) = chktmp2%p1b(:,:,:,1:numfields) + !abterms(:,:,:,1:numfields) = chktmp2%p1b(:,:,:,1:numfields) Call cheby_info%destroy() - Call chktmp2%deconstruct('p1a') - Call chktmp2%deconstruct('p1b') + !Call chktmp2%deconstruct('p1a') + !Call chktmp2%deconstruct('p1b') Deallocate(radius_old) Else ! Rayleigh doesn't currently support degrading radial resolution--exit now If (my_rank .eq. 0) Then @@ -676,13 +694,19 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Stop Endif - Else - - ! Interpolation is complete, now we just copy into the other arrays - fields(:,:,:,1:numfields) = chktmp%p1b(:,:,:,1:numfields) - abterms(:,:,:,1:numfields) = chktmp%p1b(:,:,:,numfields+1:numfields*2) + Endif + ! Interpolation is complete, now we just copy into the other arrays + fields(:,:,:,1:numfields) = chktmp%p1b(:,:,:,1:numfields) + If (version .ge. 2) Then + Call gridcp%From_Spectral(abterms_cheby%p1b,abterms) + Call abterms_cheby%deconstruct('p1b') + Else + Call gridcp%From_Spectral(chktmp2%p1a,abterms) + Call chktmp2%deconstruct('p1a') + !abterms(:,:,:,1:numfields) = chktmp%p1b(:,:,:,numfields+1:numfields*2) Endif + Call chktmp%deconstruct('p1b') DeAllocate(old_radius) From 2044e6cc70ed352f90c6192223b21cbdb07f9f9c Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Mon, 13 Jul 2026 14:38:33 -0600 Subject: [PATCH 03/28] Small adjustments to text --- src/Physics/Checkpointing.F90 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Physics/Checkpointing.F90 b/src/Physics/Checkpointing.F90 index f3435da9c..91d67a78e 100755 --- a/src/Physics/Checkpointing.F90 +++ b/src/Physics/Checkpointing.F90 @@ -188,7 +188,6 @@ Subroutine Write_Checkpoint(abterms,iteration,dt,new_dt,elapsed_time, input_file checkfile = Trim(my_path)//trim(checkpoint_prefix)//'/'//trim(checkpoint_suffix(i)) Call checkpoint_buffer%cache_data_spectral(chktmp%s2a,i) Call checkpoint_buffer%write_data(filename=checkfile, clear_existing = .true.) - Enddo Call chktmp%deconstruct('s2a') @@ -638,8 +637,8 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) If ((n_r_old .lt. n_r) .and. (version .lt. 2) ) Then - ! Prior to version 2, the AB terms were stored in radial-physical space rather - ! than in radial-Chebyshev space. + ! Prior to version 2, the AB terms were stored in radial-physical + ! space rather than in radial-Chebyshev space. ! Before proceeding, we need to convert to Chebyshev space. ! During the read-in process above, old fields were stored in ! indices 1:n_r_old of first dimension of chktmp%p1b. From 01e860cf1973ef47e7b488ac2bb8754ea1afcdee Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Tue, 14 Jul 2026 13:36:02 -0600 Subject: [PATCH 04/28] Updates/bug fixes to checkpoints -- tests working --- src/Physics/Checkpointing.F90 | 146 ++++++++++++++++------------------ 1 file changed, 68 insertions(+), 78 deletions(-) diff --git a/src/Physics/Checkpointing.F90 b/src/Physics/Checkpointing.F90 index 91d67a78e..b6846acc7 100755 --- a/src/Physics/Checkpointing.F90 +++ b/src/Physics/Checkpointing.F90 @@ -261,7 +261,7 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Real*8, Intent(InOut) :: fields(:,:,:,:), abterms(:,:,:,:) Integer :: n_r_old, l_max_old, grid_type_old Integer :: i, ierr, mp, lb,ub, ab_offset - Integer :: old_pars(7 + nsubmax), fcount(3,2), version + Integer :: old_pars(8 + nsubmax), fcount(3,2), version Integer :: last_iter, last_auto, endian_tag, funit Integer*8 :: found_bytes, expected_bytes, n_r_old_big, l_max_old_big Integer :: read_magnetism = 0, read_hydro = 0 @@ -276,7 +276,7 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Logical :: legacy_format, fexist Integer :: ncheby_old(1:nsubmax) Integer :: idom, n_r_loc, n_r_old_loc, irmin, irmax, irmin_old, irmax_old - Logical :: first_time_interpolating = .True. + Logical :: first_time_interpolating = .True., interpolate=.False. read_hydro = read_pars(1) read_magnetism = read_pars(2) @@ -373,6 +373,7 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) If (legacy_format .or. (endian_tag .eq. 314)) Then If (.not. legacy_format) Then Read(funit)version + Write(6,*)'Current Checkpoint Version: ', version Read(funit)n_r_old Else Read(funit)n_r_old @@ -440,7 +441,8 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) old_pars(2) = grid_type_old old_pars(3) = l_max_old old_pars(7) = ierr - old_pars(8:8+nsubmax-1) = ncheby_old(1:nsubmax) + old_pars(8) = version + old_pars(9:9+nsubmax-1) = ncheby_old(1:nsubmax) dt_pars(1) = dt dt_pars(2) = new_dt dt_pars(3) = checkpoint_time @@ -457,13 +459,14 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) l_max_old = old_pars(3) checkpoint_iter = old_pars(4) last_auto = old_pars(5) + version = old_pars(8) If (old_pars(6) .eq. 2) Then under_slash='_' legacy_format=.true. Endif - ncheby_old(1:nsubmax) = old_pars(8:8+nsubmax-1) + ncheby_old(1:nsubmax) = old_pars(9:9+nsubmax-1) If (old_pars(7) .ne. 0) Then ! Something is wrong with this checkpoint. @@ -565,6 +568,10 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) DeAllocate(rinds) DeAllocate(gpars) Endif + + ! Load the state vector fields and the AB terms. + ! During the read-in process below, old fields are stored in + ! indices 1:n_r_old of first dimension of chktmp%p1b. Do i = 1, numfields*2 If (read_var(i) .eq. 1) Then checkfile = trim(checkpoint_prefix)//under_slash//trim(checkpoint_suffix(i)) @@ -572,17 +579,8 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Call checkpoint_inbuffer%grab_data_spectral(chktmp%s2b,i) Endif Enddo - Call chktmp%reform() ! move to p1b - If (version .ge. 2) Then - ! For version 2+, the AB terms are stored in Chebyshev space - Call abterms_cheby%construct('p1b') - abterms_cheby%config='p1b' - abterms_cheby%p1b(:,:,:,1:numfields)=chktmp%p1b(:,:,:,numfields+1:numfields*2) - Endif - - If (.not. legacy_format) Then ! Load the boundary values array @@ -608,42 +606,40 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Endif - If (ndomains .eq. 1) Then ! Keep everything the same as it was before - ! interpolation in multiple domains was enabled - - ! NOW, if n_r_old and grid_type_old are the same, we can copy chtkmp%p1b into abterms and - ! fields. Otherwise, we need to interpolate onto the current grid - ! When we change the checkpointing format, should also store AB terms in cheby-space - If ((n_r_old .ne. n_r) .or. (grid_type_old .ne. grid_type) ) Then - ! Interpolate - ! We will assume the user kept the same radial domain bounds. - ! If they have not, this will end badly. - If (my_rank .eq. 0) Then - Call stdout%print(' ') - Call stdout%print('------ Radial grid has changed.') - Call stdout%print('------ Interpolating onto new grid.') - Write(szstr,'(i13)')grid_type_old - Call stdout%print('------ Old grid_type: '//TRIM(szstr)) - Write(szstr,'(i13)')grid_type - Call stdout%print('------ Current grid_type: '//TRIM(szstr)) - Write(szstr,'(i13)')n_r_old - Call stdout%print('------ Old N_R: '//TRIM(szstr)) - Write(szstr,'(i13)')n_r - Call stdout%print('------ Current N_R: '//TRIM(szstr)) - Call stdout%print(' ') - Endif - - - - If ((n_r_old .lt. n_r) .and. (version .lt. 2) ) Then + If (version .ge. 3) Then + ! For version 3+, the AB terms are stored in Chebyshev space + Call abterms_cheby%construct('p1b') + abterms_cheby%config='p1b' + abterms_cheby%p1b = 0d0 + ub = Min(n_r,n_r_old) + abterms_cheby%p1b(1:ub,:,:,1:numfields)=chktmp%p1b(1:ub,:,:,numfields+1:numfields*2) + Write(6,*)'In first 3-branch' + Endif - ! Prior to version 2, the AB terms were stored in radial-physical - ! space rather than in radial-Chebyshev space. - ! Before proceeding, we need to convert to Chebyshev space. - ! During the read-in process above, old fields were stored in - ! indices 1:n_r_old of first dimension of chktmp%p1b. + If (ndomains .eq. 1) Then + ! Regardless of the checkpoint version or resolution, we can load the state vector + ! fields directly from the checkpoint buffer because that portion of the buffer is + ! already in the expected Chebyshev format. + ub = Min(n_r,n_r_old) + fields = 0d0 + fields(1:ub,:,:,1:numfields) = chktmp%p1b(1:ub,:,:,1:numfields) + ! For checkpoint version 3 and later, the AB terms are also stored in + ! Chebyshev space, but they need to be in grid space. + ! Even if the resolution changes, no further work needs to be done. + If (version .ge. 3) Then + !Need to convert from Chebyshev space to physical space + Call gridcp%From_Spectral(abterms_cheby%p1b,abterms) + Call abterms_cheby%deconstruct('p1b') + Else + ! For checkpoint versions 1 and 2, AB terms were stored in physical space. + ! If the grid resolution changed, we need to perform a Chebyshev interpolation. + ! If not, we can load directly from the checkpoint buffer. + If (n_r_old .eq. n_r) Then + abterms(:,:,:,1:numfields) = chktmp%p1b(:,:,:,numfields+1:numfields*2) + Else + Write(6,*)'Inside interpolation branch' ! (1) Initialize the old Chebyshev grid. Allocate(radius_old(1:n_r_old)) Call cheby_info%Init(radius_old,rmin,rmax) ! rmax and rmin cannot change @@ -654,7 +650,7 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Call chktmp2%init(field_count = fcount, config = 'p1a') Call chktmp2%construct('p1a') chktmp2%p1a(:,:,:,:) = 0.0d0 - lb = lbound(chktmp%p1b,3) + lb = lbound(chktmp%p1b,3) ! describes bounds of lm mode index ub = ubound(chktmp%p1b,3) Allocate(tempfield1(1:n_r_old,1:2,lb:ub,1)) Allocate(tempfield2(1:n_r_old,1:2,lb:ub,1)) @@ -671,43 +667,37 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) ! Grab Chebyshev coefficients up to old n_max chktmp2%p1a(1:n_r_old,:,:,i) = tempfield2(1:n_r_old,:,:,1) Enddo - DeAllocate(tempfield1,tempfield2) - - - !Call chktmp2%construct('p1b') - !Normal transform(p1a,p1b) - !Call gridcp%From_Spectral(chktmp2%p1a,chktmp2%p1b) + Call gridcp%From_Spectral(chktmp2%p1a,abterms) - !abterms(:,:,:,1:numfields) = chktmp2%p1b(:,:,:,1:numfields) + ! clean up + DeAllocate(tempfield1,tempfield2) Call cheby_info%destroy() - !Call chktmp2%deconstruct('p1a') - !Call chktmp2%deconstruct('p1b') Deallocate(radius_old) - Else ! Rayleigh doesn't currently support degrading radial resolution--exit now - If (my_rank .eq. 0) Then - Call stdout%print('ERROR: Rayleigh currently does not support degrading radial resolution.') - Call stdout%print('Now exiting') - Call stdout%partial_flush() - Endif - Call pfi%exit() - Stop - Endif + Call chktmp2%deconstruct('p1a') - Endif + Endif ! (n_r_old .eq. n_r) - ! Interpolation is complete, now we just copy into the other arrays - fields(:,:,:,1:numfields) = chktmp%p1b(:,:,:,1:numfields) - If (version .ge. 2) Then - Call gridcp%From_Spectral(abterms_cheby%p1b,abterms) - Call abterms_cheby%deconstruct('p1b') - Else - Call gridcp%From_Spectral(chktmp2%p1a,abterms) - Call chktmp2%deconstruct('p1a') - !abterms(:,:,:,1:numfields) = chktmp%p1b(:,:,:,numfields+1:numfields*2) + Endif ! (version .ge. 3) + + ! clean up + Call chktmp%deconstruct('p1b') + Deallocate(old_radius) + + ! Finally, print a message if we interpolated + If ( (my_rank .eq. 0) .and. (n_r .ne. n_r_old) ) Then + Call stdout%print(' ') + Call stdout%print('------ Radial grid has changed.') + Call stdout%print('------ Interpolating onto new grid.') + Write(szstr,'(i13)')grid_type_old + Call stdout%print('------ Old grid_type: '//TRIM(szstr)) + Write(szstr,'(i13)')grid_type + Call stdout%print('------ Current grid_type: '//TRIM(szstr)) + Write(szstr,'(i13)')n_r_old + Call stdout%print('------ Old N_R: '//TRIM(szstr)) + Write(szstr,'(i13)')n_r + Call stdout%print('------ Current N_R: '//TRIM(szstr)) + Call stdout%print(' ') Endif - - Call chktmp%deconstruct('p1b') - DeAllocate(old_radius) Else ! ndomains > 1: we need to loop over domains and (maybe) interpolate From ca398781aeec3dc66a6ebf0890143002b8ef89c5 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Tue, 14 Jul 2026 15:35:01 -0600 Subject: [PATCH 05/28] Adding error messages. --- src/Physics/Checkpointing.F90 | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Physics/Checkpointing.F90 b/src/Physics/Checkpointing.F90 index b6846acc7..86f1b9f0a 100755 --- a/src/Physics/Checkpointing.F90 +++ b/src/Physics/Checkpointing.F90 @@ -613,7 +613,6 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) abterms_cheby%p1b = 0d0 ub = Min(n_r,n_r_old) abterms_cheby%p1b(1:ub,:,:,1:numfields)=chktmp%p1b(1:ub,:,:,numfields+1:numfields*2) - Write(6,*)'In first 3-branch' Endif @@ -636,9 +635,11 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) ! For checkpoint versions 1 and 2, AB terms were stored in physical space. ! If the grid resolution changed, we need to perform a Chebyshev interpolation. ! If not, we can load directly from the checkpoint buffer. + ! Due to the storage convention for versions 1 and 2, interpolation + ! can only be carried out if n_r_old < n_r. If (n_r_old .eq. n_r) Then abterms(:,:,:,1:numfields) = chktmp%p1b(:,:,:,numfields+1:numfields*2) - Else + Else If (n_r_old .lt. n_r) Then Write(6,*)'Inside interpolation branch' ! (1) Initialize the old Chebyshev grid. Allocate(radius_old(1:n_r_old)) @@ -674,7 +675,16 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Call cheby_info%destroy() Deallocate(radius_old) Call chktmp2%deconstruct('p1a') + Else + If (my_rank .eq. 0) Then + Call stdout%print('****************** ERROR ******************') + Call stdout%print('ERROR: Rayleigh does not support degrading radial resolution') + Call stdout%print('for checkpoints generated using version 1.3.0 or earlier.') + Call stdout%partial_flush() + Endif + Call pfi%exit() + Stop Endif ! (n_r_old .eq. n_r) Endif ! (version .ge. 3) @@ -697,6 +707,14 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Write(szstr,'(i13)')n_r Call stdout%print('------ Current N_R: '//TRIM(szstr)) Call stdout%print(' ') + If (n_r_old .gt. n_r) Then + Call stdout%print('------ ******************* WARNING *****************************') + Call stdout%print('------ Boundary conditions are not formally satisfied by the') + Call stdout%print('------ checkpoint save state when N_R is decreased upon restart.') + Call stdout%print('------ Drastically degrading the radial resolution may lead') + Call stdout%print('------ to unexpected results.') + Call stdout%print(' ') + Endif Endif Else ! ndomains > 1: we need to loop over domains and (maybe) interpolate From ae345f327b37ce0d248042b40b69e04b688e611d Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Tue, 14 Jul 2026 16:43:14 -0600 Subject: [PATCH 06/28] Added error condition and message for degrading radial resolution using old checkpoint format. --- src/Physics/Checkpointing.F90 | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Physics/Checkpointing.F90 b/src/Physics/Checkpointing.F90 index 86f1b9f0a..5df0e3ba4 100755 --- a/src/Physics/Checkpointing.F90 +++ b/src/Physics/Checkpointing.F90 @@ -677,10 +677,18 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Call chktmp2%deconstruct('p1a') Else If (my_rank .eq. 0) Then - Call stdout%print('****************** ERROR ******************') - Call stdout%print('ERROR: Rayleigh does not support degrading radial resolution') - Call stdout%print('for checkpoints generated using version 1.3.0 or earlier.') - + Call stdout%print(' ') + Call stdout%print('******************************** ERROR *********************************') + Call stdout%print('------ Rayleigh does not support degrading radial resolution') + Call stdout%print('------ for checkpoints generated using version 1.3.0 or earlier.') + Call stdout%print(' ') + Call stdout%print('------ In order to degrade the radial resolution of this model:') + Call stdout%print('------ (1) Restart from the current checkpoint using this version of') + Call stdout%print('------ Rayleigh and maintain the current radial resolution.') + Call stdout%print('------ (2) Generate a new checkpoint at the current resolution.') + Call stdout%print('------ This checkpoint will be saved using the updated format.') + Call stdout%print('------ (3) Restart using the new checkpoint and desired radial resolution.') + Call stdout%print(' ') Call stdout%partial_flush() Endif Call pfi%exit() From 34273e74456b003b34caac7373f75f127e3bf991 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Tue, 14 Jul 2026 17:17:20 -0600 Subject: [PATCH 07/28] Removed unused interpolate variable from read_checkpoint --- src/Physics/Checkpointing.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Physics/Checkpointing.F90 b/src/Physics/Checkpointing.F90 index 5df0e3ba4..fdde1a4a0 100755 --- a/src/Physics/Checkpointing.F90 +++ b/src/Physics/Checkpointing.F90 @@ -276,7 +276,7 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Logical :: legacy_format, fexist Integer :: ncheby_old(1:nsubmax) Integer :: idom, n_r_loc, n_r_old_loc, irmin, irmax, irmin_old, irmax_old - Logical :: first_time_interpolating = .True., interpolate=.False. + Logical :: first_time_interpolating = .True. read_hydro = read_pars(1) read_magnetism = read_pars(2) From 7a683950934ec90fbdb76a6b370744e492382f45 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Thu, 23 Jul 2026 11:42:22 -0600 Subject: [PATCH 08/28] Updating workflow to also build Rayleigh 1.3 for testing --- .github/workflows/main.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5164febe2..29dc59458 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,6 +44,17 @@ jobs: cd "$GITHUB_WORKSPACE" make clear_ipynb && git diff --exit-code --name-only make doc + - name: Build Rayleigh 1.3 + # Need this for testing checkpoint format compatibility + run: | + git checkout rayleigh-1.3.0 + ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' + + make fdeps && git diff --exit-code + make -j + make install + mv ./bin/rayleigh.dbg ./bin/rayleigh.v13 + git checkout main - name: Build Rayleigh # Now build Rayleigh itself run: | From 1c88131ef7273f8c1772df4c54edf428b20e5285 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Thu, 23 Jul 2026 11:56:00 -0600 Subject: [PATCH 09/28] updating workflow --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 29dc59458..b7a9638a0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,7 +47,8 @@ jobs: - name: Build Rayleigh 1.3 # Need this for testing checkpoint format compatibility run: | - git checkout rayleigh-1.3.0 + git clone -b rayleigh-1.3.0 https://github.com/geodynamics/Rayleigh.git + cd Rayleigh ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' make fdeps && git diff --exit-code From 480ec04983032ff2b1190fc87003caf2fd032c89 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Thu, 23 Jul 2026 12:05:06 -0600 Subject: [PATCH 10/28] Another update --- .github/workflows/main.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b7a9638a0..081be7935 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,16 +48,19 @@ jobs: # Need this for testing checkpoint format compatibility run: | git clone -b rayleigh-1.3.0 https://github.com/geodynamics/Rayleigh.git - cd Rayleigh + ls + pwd + mv Rayleigh Rayleigh_1.3 + cd Rayleigh_1.3 ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' make fdeps && git diff --exit-code make -j - make install - mv ./bin/rayleigh.dbg ./bin/rayleigh.v13 - git checkout main + make install + cd .. + ln -s Rayleigh_1.3/rayleigh.dbg ./bin/rayleigh.v13 - name: Build Rayleigh - # Now build Rayleigh itself + # Now build the current Rayleigh run: | ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' From a836a2c7b2b4736c0c378c62b1a121b31f4da4bb Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Thu, 23 Jul 2026 12:11:46 -0600 Subject: [PATCH 11/28] Still trying to get 1.3 to build for testing --- .github/workflows/main.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 081be7935..873d9809d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -48,12 +48,11 @@ jobs: # Need this for testing checkpoint format compatibility run: | git clone -b rayleigh-1.3.0 https://github.com/geodynamics/Rayleigh.git - ls - pwd mv Rayleigh Rayleigh_1.3 cd Rayleigh_1.3 + ls + pwd ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' - make fdeps && git diff --exit-code make -j make install From 508fd1c9272dc6f77dfc555e529addefbe16db35 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Thu, 23 Jul 2026 12:21:16 -0600 Subject: [PATCH 12/28] Debugging workflow --- .github/workflows/main.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 873d9809d..395fa421f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,13 +49,14 @@ jobs: run: | git clone -b rayleigh-1.3.0 https://github.com/geodynamics/Rayleigh.git mv Rayleigh Rayleigh_1.3 - cd Rayleigh_1.3 ls - pwd - ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' - make fdeps && git diff --exit-code - make -j - make install + pwd + cd Rayleigh_1.3 + + #./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' + #make fdeps && git diff --exit-code + #make -j + #make install cd .. ln -s Rayleigh_1.3/rayleigh.dbg ./bin/rayleigh.v13 - name: Build Rayleigh From 46c5b1035fe472e26fee79434b7059e9bf03d65e Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Thu, 23 Jul 2026 12:34:42 -0600 Subject: [PATCH 13/28] Still testing workflows --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 395fa421f..d2c248b94 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,7 +58,7 @@ jobs: #make -j #make install cd .. - ln -s Rayleigh_1.3/rayleigh.dbg ./bin/rayleigh.v13 + #ln -s Rayleigh_1.3/rayleigh.dbg ./bin/rayleigh.v13 - name: Build Rayleigh # Now build the current Rayleigh run: | From c93dd3499a304e4dc514753cbe05af86a4beccd3 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Thu, 23 Jul 2026 13:14:43 -0600 Subject: [PATCH 14/28] Increased size of abterms_cheby by 1 in last dimension --- src/Physics/Checkpointing.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Physics/Checkpointing.F90 b/src/Physics/Checkpointing.F90 index fdde1a4a0..f37d77884 100755 --- a/src/Physics/Checkpointing.F90 +++ b/src/Physics/Checkpointing.F90 @@ -141,6 +141,7 @@ Subroutine Initialize_Checkpointing() ! Buffer to hold Adams-Bashforth terms in Chebyshev format nfs(:) = numfields + if (magnetism) nfs(:) = numfields+1 ! magnetic mode has one extra field space used to combined derivatives Call abterms_cheby%init(field_count = nfs, config = 'p1a') End Subroutine Initialize_Checkpointing From 72a0caa9933ded869aa5054610b61b547ce0dc88 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Thu, 23 Jul 2026 13:33:16 -0600 Subject: [PATCH 15/28] Added some additional testing labels to help see failing tests better --- .github/workflows/main.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d2c248b94..65bae7df5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -67,12 +67,13 @@ jobs: make fdeps && git diff --exit-code make -j make install - - name: Test + - name: Chebyshev Test run: | #Primary test (Chebyshev Mode) cd "$GITHUB_WORKSPACE"/tests/c2001_case0 mpirun -np 4 ../../bin/rayleigh.dbg - + - name: Finite-Difference Test + run: | # Finite-difference test (uniform grid) cd "$GITHUB_WORKSPACE"/tests/c2001_case0_FD_uniform mpirun -np 4 ../../bin/rayleigh.dbg @@ -84,19 +85,23 @@ jobs: # J2011 steady mhd test # cd "$GITHUB_WORKSPACE"/tests/j2011_steady_mhd_minimal # mpirun -np 4 ../../bin/rayleigh.dbg - + - name: Generic Input Tests + run: | # Generic input test cd "$GITHUB_WORKSPACE" sh ./tests/generic_input/run_test.sh - + - name: Chi Scalar Tests + run: | # chi scalar test cd "$GITHUB_WORKSPACE" sh ./tests/chi_scalar/run_test.sh - + - name: Custom Reference Test + run: | # custom reference state test cd "$GITHUB_WORKSPACE" sh ./tests/custom_reference/run_test.sh - + - name: Coupled BC Test + run: | # coupled bc test cd "$GITHUB_WORKSPACE" sh ./tests/coupled_bcs/run_test.sh From 77c19b1f25a1fee5565c404db9c51aaac5f70632 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Thu, 23 Jul 2026 14:25:56 -0600 Subject: [PATCH 16/28] Rolling back changes to the main workflow file --- .github/workflows/main.yml | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1a96710dd..47e907271 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,36 +44,20 @@ jobs: cd "$GITHUB_WORKSPACE" make clear_ipynb && git diff --exit-code --name-only make doc - - name: Build Rayleigh 1.3 - # Need this for testing checkpoint format compatibility - run: | - git clone -b rayleigh-1.3.0 https://github.com/geodynamics/Rayleigh.git - mv Rayleigh Rayleigh_1.3 - ls - pwd - cd Rayleigh_1.3 - - #./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' - #make fdeps && git diff --exit-code - #make -j - #make install - cd .. - #ln -s Rayleigh_1.3/rayleigh.dbg ./bin/rayleigh.v13 - name: Build Rayleigh - # Now build the current Rayleigh + # Now build Rayleigh itself run: | ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' make fdeps && git diff --exit-code make -j make install - - name: Chebyshev Test + - name: Test run: | #Primary test (Chebyshev Mode) cd "$GITHUB_WORKSPACE"/tests/c2001_case0 mpirun -np 4 ../../bin/rayleigh.dbg - - name: Finite-Difference Test - run: | + # Finite-difference test (uniform grid) cd "$GITHUB_WORKSPACE"/tests/c2001_case0_FD_uniform mpirun -np 4 ../../bin/rayleigh.dbg @@ -85,23 +69,18 @@ jobs: # J2011 steady mhd test # cd "$GITHUB_WORKSPACE"/tests/j2011_steady_mhd_minimal # mpirun -np 4 ../../bin/rayleigh.dbg - - name: Generic Input Tests - run: | + # Generic input test cd "$GITHUB_WORKSPACE" sh ./tests/generic_input/run_test.sh - - name: Chi Scalar Tests - run: | + # chi scalar test cd "$GITHUB_WORKSPACE" sh ./tests/chi_scalar/run_test.sh - - name: Custom Reference Test - run: | + # custom reference state test cd "$GITHUB_WORKSPACE" sh ./tests/custom_reference/run_test.sh - - name: Coupled BC Test - run: | # chi custom reference state test cd "$GITHUB_WORKSPACE" From 3e38871580aa9352228b8f810fbd875064c705db Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Thu, 23 Jul 2026 15:06:00 -0600 Subject: [PATCH 17/28] Corrected size of "return-trip" buffers in Fields.F90. n_active_scalars and n_passive_scalars were getting double counted --- src/Physics/Fields.F90 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Physics/Fields.F90 b/src/Physics/Fields.F90 index 72c99c973..730a7f593 100755 --- a/src/Physics/Fields.F90 +++ b/src/Physics/Fields.F90 @@ -452,17 +452,17 @@ Subroutine Initialize_Field_Structure() ! These following code should pretty much never be modified by the user. if (.not. magnetism) then - wsfcount(1,2) = n_equations + n_active_scalars + n_passive_scalars - wsfcount(2,2) = n_equations + n_active_scalars + n_passive_scalars - wsfcount(3,2) = n_equations + n_active_scalars + n_passive_scalars + wsfcount(1,2) = n_equations + wsfcount(2,2) = n_equations + wsfcount(3,2) = n_equations else emfr = avar emftheta = cvar emfphi = avar+1 ! seven RHS's (plus scalars) go back for the solve (1 field is differentiated and combined at the end) - wsfcount(1,2) = n_equations + 1 + n_active_scalars + n_passive_scalars - wsfcount(2,2) = n_equations + 1 + n_active_scalars + n_passive_scalars - wsfcount(3,2) = n_equations + 1 + n_active_scalars + n_passive_scalars + wsfcount(1,2) = n_equations + 1 + wsfcount(2,2) = n_equations + 1 + wsfcount(3,2) = n_equations + 1 endif From 3018a2839e0a73158f9a43ac70adf5a91b82006f Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Fri, 24 Jul 2026 13:55:06 -0600 Subject: [PATCH 18/28] Initial draft of checkpoint tests. --- .github/workflows/main.yml | 41 +++++++++--- tests/checkpoint/compare_check.py | 58 ++++++++++++++++ tests/checkpoint/old_format/main_input | 77 +++++++++++++++++++++ tests/checkpoint/run_test.sh | 92 ++++++++++++++++++++++++++ 4 files changed, 260 insertions(+), 8 deletions(-) create mode 100644 tests/checkpoint/compare_check.py create mode 100644 tests/checkpoint/old_format/main_input create mode 100644 tests/checkpoint/run_test.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 47e907271..60699ac2a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,12 +52,31 @@ jobs: make fdeps && git diff --exit-code make -j make install - - name: Test + - name: Build Rayleigh v 1.3 + # Build 1.3 for testing checkpoint compatibility + run: | + + mv Rayleigh Rayleigh_1.3 + cd Rayleigh_1.3 + ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' + + make fdeps && git diff --exit-code + make -j + make install + - name: Checkpoint Test + run: | + # coupled bc test + cd "$GITHUB_WORKSPACE" + cd ./tests/checkpoint + sh ./run_test.sh + + - name: Chebyshev Test run: | #Primary test (Chebyshev Mode) cd "$GITHUB_WORKSPACE"/tests/c2001_case0 mpirun -np 4 ../../bin/rayleigh.dbg - + - name: Finite-difference Test + run: | # Finite-difference test (uniform grid) cd "$GITHUB_WORKSPACE"/tests/c2001_case0_FD_uniform mpirun -np 4 ../../bin/rayleigh.dbg @@ -69,27 +88,33 @@ jobs: # J2011 steady mhd test # cd "$GITHUB_WORKSPACE"/tests/j2011_steady_mhd_minimal # mpirun -np 4 ../../bin/rayleigh.dbg - + - name: Generic Input Test + run: | # Generic input test cd "$GITHUB_WORKSPACE" sh ./tests/generic_input/run_test.sh - + - name: Chi Scalar Test + run: | # chi scalar test cd "$GITHUB_WORKSPACE" sh ./tests/chi_scalar/run_test.sh - + - name: Custom Reference Test + run: | # custom reference state test cd "$GITHUB_WORKSPACE" sh ./tests/custom_reference/run_test.sh - + - name: Chi Custom Reference Test + run: | # chi custom reference state test cd "$GITHUB_WORKSPACE" sh ./tests/chi_custom_reference/run_test.sh - + - name: Coupled BC Test + run: | # coupled bc test cd "$GITHUB_WORKSPACE" sh ./tests/coupled_bcs/run_test.sh - + - name: Vforce Diagnostics Test + run: | # vforce diagnostics regression test cd "$GITHUB_WORKSPACE" sh ./tests/vforce_diagnostics/run_test.sh diff --git a/tests/checkpoint/compare_check.py b/tests/checkpoint/compare_check.py new file mode 100644 index 000000000..ec83b6832 --- /dev/null +++ b/tests/checkpoint/compare_check.py @@ -0,0 +1,58 @@ +from rayleigh_diagnostics import G_Avgs, SPH_Modes +import numpy as np + +def chisq(a1,a2): + b = (a1-a2)**2 + c = np.sum(b)/np.sum(a1*a1) + return c**0.5 + +dirs = ['old_format', 'new_format', 'new_format_same_res', 'new_format_up_res', 'new_format_down_res'] +iters = ['00000100', '00000150', '00000200','00000250'] +tols = [1e-10, 1e-10, 1e-2, 1e-2] +ndirs = len(dirs) +for i in range(1,ndirs): + #print('=========================') + chisqs = [] + old = dirs[i-1] + new = dirs[i] + istring = iters[i-1] + #print(old,new,istring) + + sphm_o = SPH_Modes(istring,path=old+'/SPH_Modes/') + sphm_n = SPH_Modes(istring,path=new+'/SPH_Modes/') + ga_o = G_Avgs(istring,path=old+'/G_Avgs/') + ga_n = G_Avgs(istring,path=new+'/G_Avgs/') + + mx_ref = np.max([sphm_o.vals.real**2,sphm_o.vals.imag**2])**0.5 + for j, l in enumerate(sphm_o.lvals): + for m in range(0,l+1): + val_o = sphm_o.vals[m,j,0,0,:] + val_n = sphm_n.vals[m,j,0,0,:] + c = chisq(val_o.real,val_n.real) + + mx = np.max([val_o.real**2,val_n.imag**2])**0.5 + chisqs.append(c*mx/mx_ref) + + if (m > 0): # m = 0 has no imaginary component + c = chisq(val_o.imag,val_n.imag) + chisqs.append(c*mx/mx_ref) + if ((l == 4000) and (m ==4)): + fig,ax = plt.subplots(ncols=2,figsize=(10,5)) + ax[0].plot(val_o.real) + ax[0].plot(val_n.real) + ax[1].plot(val_o.imag) + ax[1].plot(val_n.imag) + plt.show() + sph_csq = np.max(chisqs) + + val_o = ga_o.vals[:,0] + val_n = ga_n.vals[:,0] + ga_csq = chisq(val_o,val_n) + + #print('sph modes: ', sph_csq) + #print('G_Avgs: ', ga_csq) + if ( (ga_csq > tols[i-1]) or (sph_csq > tols[i-1]) ): + print('Checkpoint Test Error: Time series do not agree.') + exit(1) +print('Checkpoint Test Passed') +exit(0) diff --git a/tests/checkpoint/old_format/main_input b/tests/checkpoint/old_format/main_input new file mode 100644 index 000000000..66f3d4c8d --- /dev/null +++ b/tests/checkpoint/old_format/main_input @@ -0,0 +1,77 @@ +&problemsize_namelist + n_r=48 + n_theta = 64 + nprow = 2 + npcol = 2 + aspect_ratio = 0.35d0 + shell_depth = 1.0d0 +/ +&numerical_controls_namelist +/ +&physical_controls_namelist +! benchmark_mode = 1 +! benchmark_integration_interval = 100 +! benchmark_report_interval = 5000 + rotation = .True. + magnetism = .false. + viscous_heating = .false. + ohmic_heating = .false. + advect_reference_state = .false. +/ +&temporal_controls_namelist + max_time_step = 1.0d-4 + max_iterations = 200 + checkpoint_interval = 25 + cflmin = 0.4d0 + cflmax = 0.6d0 +/ +&io_controls_namelist +/ +&output_namelist + +! New equatorial slices output + +sph_mode_ell = 2,4,8, 32 +sph_mode_levels = 1 ! outer boundary +sph_mode_values = 507 ! radial entropy gradient +sph_mode_frequency = 10 +sph_mode_nrec = 5 + +globalavg_values = 401 ! kinetic energy +globalavg_frequency = 10 +globalavg_nrec = 5 + + +full3d_values = 64 ! temperature +full3d_frequency = 9000000 +/ + +&Boundary_Conditions_Namelist +no_slip_boundaries = .true. +strict_L_Conservation = .false. +dtdr_bottom = 0.0d0 +T_Top = 0.0d0 +T_Bottom = 1.0d0 +fix_tvar_top = .true. +fix_tvar_bottom = .true. +/ +&Initial_Conditions_Namelist +init_type=7 ! Benchmark init +temp_amp = 1.0d-3 +temp_w = 0.01d4 +restart_iter=-1 +conductive_profile=.true. +/ +&Test_Namelist +/ +&Reference_Namelist +Ekman_Number = 1.0d-3 +Rayleigh_Number = 1.0d5 +Prandtl_Number = 1.0d0 +Magnetic_Prandtl_Number = 5.0d0 +reference_type = 1 +heating_type = 0 ! No heating +gravity_power = 1.0d0 ! g ~ radius +/ +&Transport_Namelist +/ diff --git a/tests/checkpoint/run_test.sh b/tests/checkpoint/run_test.sh new file mode 100644 index 000000000..5fe61e994 --- /dev/null +++ b/tests/checkpoint/run_test.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +export RA_ROOT=../../.. +export rayleigh_v13=$RA_ROOT/Rayleigh_1.3/bin/rayleigh.dbg +export rayleigh_dbg=.$RA_ROOT/bin/rayleigh.dbg + +#cd tests/checkpoint +cp $RA_ROOT/post_processing/rayleigh_diagnostics.py . +# Generate a checkpoint in the old format using Rayleigh v1.3 +cd old_format +mpirun -np 4 $rayleigh_v13 -niter 100 +cd .. + +# Restart from the old checkpoint format at increased resolution using Rayleigh 1.3. +lastdir=old_format +newdir=old_format_up + +mkdir $newdir +mkdir $newdir/Checkpoints +cp $lastdir/main_input $newdir/. +cp -r $lastdir/Checkpoints/00000050 $newdir/Checkpoints/. + +cd $newdir +sed -i 's/n_r=48/n_r=64/g' main_input +sed -i 's/init_type=7/init_type=-1/g' main_input +sed -i 's/restart_iter=-1/restart_iter=50/g' main_input +mpirun -np 4 $rayleigh_v13 -niter 100 +cd .. + + +# Restart from the old checkpoint format at same resolution using current Rayleigh. +lastdir=old_format +newdir=new_format + +mkdir $newdir +mkdir $newdir/Checkpoints +cp $lastdir/main_input $newdir/. +cp -r $lastdir/Checkpoints/00000050 $newdir/Checkpoints/. + +cd $newdir +#sed -i 's/n_r=48/n_r=64/g' main_input +sed -i 's/init_type=7/init_type=-1/g' main_input +sed -i 's/restart_iter=-1/restart_iter=50/g' main_input +mpirun -np 4 $rayleigh_dbg -niter 100 +cd .. + + +# Restart from the new checkpoint format at same resolution. +lastdir=new_format +newdir=new_format_same_res +mkdir $newdir +mkdir $newdir/Checkpoints +cp $lastdir/main_input $newdir/. +cp -r $lastdir/Checkpoints/00000100 $newdir/Checkpoints/. + +cd $newdir +#sed -i 's/n_r=48/n_r=64/g' main_input +#sed -i 's/init_type=1/init_type=-1/g' main_input +sed -i 's/restart_iter=50/restart_iter=100/g' main_input +mpirun -np 4 $rayleigh_dbg -niter 100 +cd .. + +# Restart from the new checkpoint format at increased resolution. +lastdir=new_format_same_res +newdir=new_format_up_res +mkdir $newdir +mkdir $newdir/Checkpoints +cp $lastdir/main_input $newdir/. +cp -r $lastdir/Checkpoints/00000150 $newdir/Checkpoints/. + +cd $newdir +sed -i 's/n_r=48/n_r=64/g' main_input +#sed -i 's/init_type=1/init_type=-1/g' main_input +sed -i 's/restart_iter=100/restart_iter=150/g' main_input +mpirun -np 4 $rayleigh_dbg -niter 100 +cd .. + +# Restart from the new checkpoint format and degrade resolution. +lastdir=new_format_up_res +newdir=new_format_down_res +mkdir $newdir +mkdir $newdir/Checkpoints +cp $lastdir/main_input $newdir/. +cp -r $lastdir/Checkpoints/00000200 $newdir/Checkpoints/. + +cd $newdir +sed -i 's/n_r=64/n_r=48/g' main_input +#sed -i 's/init_type=1/init_type=-1/g' main_input +sed -i 's/restart_iter=150/restart_iter=200/g' main_input +mpirun -np 4 $rayleigh_dbg -niter 100 +cd .. + From 77cd43cfd976bb7999b0dd7788ee10ffec112ddb Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Fri, 24 Jul 2026 13:59:45 -0600 Subject: [PATCH 19/28] Added git clone command for building Rayleigh v 1.3.0 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 60699ac2a..0229345e7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,7 +55,7 @@ jobs: - name: Build Rayleigh v 1.3 # Build 1.3 for testing checkpoint compatibility run: | - + git clone -b rayleigh-1.3.0 https://github.com/geodynamics/Rayleigh.git mv Rayleigh Rayleigh_1.3 cd Rayleigh_1.3 ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' From 8df718b8020e2b01b6801fdf1e40e6e81869dfce Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Fri, 24 Jul 2026 14:08:00 -0600 Subject: [PATCH 20/28] Fixed path to Rayleigh in run_test.sh Now also calls compare_check.py --- tests/checkpoint/run_test.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/checkpoint/run_test.sh b/tests/checkpoint/run_test.sh index 5fe61e994..2a75bf021 100644 --- a/tests/checkpoint/run_test.sh +++ b/tests/checkpoint/run_test.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -export RA_ROOT=../../.. +export RA_ROOT=../.. export rayleigh_v13=$RA_ROOT/Rayleigh_1.3/bin/rayleigh.dbg -export rayleigh_dbg=.$RA_ROOT/bin/rayleigh.dbg +export rayleigh_dbg=$RA_ROOT/bin/rayleigh.dbg #cd tests/checkpoint cp $RA_ROOT/post_processing/rayleigh_diagnostics.py . @@ -90,3 +90,4 @@ sed -i 's/restart_iter=150/restart_iter=200/g' main_input mpirun -np 4 $rayleigh_dbg -niter 100 cd .. +python3 compare_check.py From d90bfab21de5754da57b6d2e9567106dc64e9e21 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Fri, 24 Jul 2026 14:14:50 -0600 Subject: [PATCH 21/28] Updated RA_ROOT path --- tests/checkpoint/run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/checkpoint/run_test.sh b/tests/checkpoint/run_test.sh index 2a75bf021..f0c6a5fc0 100644 --- a/tests/checkpoint/run_test.sh +++ b/tests/checkpoint/run_test.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -export RA_ROOT=../.. +export RA_ROOT=../../.. export rayleigh_v13=$RA_ROOT/Rayleigh_1.3/bin/rayleigh.dbg export rayleigh_dbg=$RA_ROOT/bin/rayleigh.dbg From 5d8a039acd62578052374e1b813340df5cf093ad Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Fri, 24 Jul 2026 14:23:26 -0600 Subject: [PATCH 22/28] Fixed path to rayleigh_diagnostics.py --- tests/checkpoint/run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/checkpoint/run_test.sh b/tests/checkpoint/run_test.sh index f0c6a5fc0..8626bbde5 100644 --- a/tests/checkpoint/run_test.sh +++ b/tests/checkpoint/run_test.sh @@ -5,7 +5,7 @@ export rayleigh_v13=$RA_ROOT/Rayleigh_1.3/bin/rayleigh.dbg export rayleigh_dbg=$RA_ROOT/bin/rayleigh.dbg #cd tests/checkpoint -cp $RA_ROOT/post_processing/rayleigh_diagnostics.py . +cp ../../post_processing/rayleigh_diagnostics.py . # Generate a checkpoint in the old format using Rayleigh v1.3 cd old_format mpirun -np 4 $rayleigh_v13 -niter 100 From bbccb3552fb9e88bf1bab969d3aae7a45ecac772 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Fri, 24 Jul 2026 14:32:31 -0600 Subject: [PATCH 23/28] Updated checkpoint test label. Rayleigh 1.3 now compiles with -devel flag. --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0229345e7..ff2ede17c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,14 +58,14 @@ jobs: git clone -b rayleigh-1.3.0 https://github.com/geodynamics/Rayleigh.git mv Rayleigh Rayleigh_1.3 cd Rayleigh_1.3 - ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' + ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' -devel make fdeps && git diff --exit-code make -j make install - name: Checkpoint Test run: | - # coupled bc test + # Checkpoint Test cd "$GITHUB_WORKSPACE" cd ./tests/checkpoint sh ./run_test.sh From 5ec68ef8ad24b07724fd68618965b0a93e18da34 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Fri, 24 Jul 2026 14:37:05 -0600 Subject: [PATCH 24/28] Removed devel flag. Hacked g_csq to force failure. --- tests/checkpoint/compare_check.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/checkpoint/compare_check.py b/tests/checkpoint/compare_check.py index ec83b6832..f3886a080 100644 --- a/tests/checkpoint/compare_check.py +++ b/tests/checkpoint/compare_check.py @@ -50,7 +50,8 @@ def chisq(a1,a2): ga_csq = chisq(val_o,val_n) #print('sph modes: ', sph_csq) - #print('G_Avgs: ', ga_csq) + #print('G_Avgs: ', ga_csq) + ga_csq = 44 if ( (ga_csq > tols[i-1]) or (sph_csq > tols[i-1]) ): print('Checkpoint Test Error: Time series do not agree.') exit(1) From 3e4e30a66d871cffdf1ddd2771399db7d72696cc Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Fri, 24 Jul 2026 14:44:51 -0600 Subject: [PATCH 25/28] Removing make fdeps from v1.3 build --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ff2ede17c..7920c01f8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,9 +58,9 @@ jobs: git clone -b rayleigh-1.3.0 https://github.com/geodynamics/Rayleigh.git mv Rayleigh Rayleigh_1.3 cd Rayleigh_1.3 - ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' -devel + ./configure -debian-mkl ${{ matrix.mpi }} --FFLAGS_DBG='-O0 -g -fbounds-check -ffpe-trap=invalid,zero,overflow -fbacktrace -ffixed-line-length-132 -Wall' - make fdeps && git diff --exit-code + #make fdeps && git diff --exit-code make -j make install - name: Checkpoint Test From b03239a69b3f9a23d00d67ac712b6fc14eb13601 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Fri, 24 Jul 2026 14:50:54 -0600 Subject: [PATCH 26/28] Removed g_csq hack. Added label in main.yml --- .github/workflows/main.yml | 1 + tests/checkpoint/compare_check.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7920c01f8..12f96ce14 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -55,6 +55,7 @@ jobs: - name: Build Rayleigh v 1.3 # Build 1.3 for testing checkpoint compatibility run: | + # Build Rayleigh v 1.3 git clone -b rayleigh-1.3.0 https://github.com/geodynamics/Rayleigh.git mv Rayleigh Rayleigh_1.3 cd Rayleigh_1.3 diff --git a/tests/checkpoint/compare_check.py b/tests/checkpoint/compare_check.py index f3886a080..a9d995aee 100644 --- a/tests/checkpoint/compare_check.py +++ b/tests/checkpoint/compare_check.py @@ -51,7 +51,6 @@ def chisq(a1,a2): #print('sph modes: ', sph_csq) #print('G_Avgs: ', ga_csq) - ga_csq = 44 if ( (ga_csq > tols[i-1]) or (sph_csq > tols[i-1]) ): print('Checkpoint Test Error: Time series do not agree.') exit(1) From 38791f639bf721085729c3838a627e819ac477f3 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Tue, 11 Aug 2026 11:58:50 -0600 Subject: [PATCH 27/28] Added logic to prevent intererence of new checkpoint format with finite-difference mode --- src/Physics/Checkpointing.F90 | 36 +++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/Physics/Checkpointing.F90 b/src/Physics/Checkpointing.F90 index 1fdbbef12..2c8e71d4d 100755 --- a/src/Physics/Checkpointing.F90 +++ b/src/Physics/Checkpointing.F90 @@ -174,14 +174,18 @@ Subroutine Write_Checkpoint(abterms,iteration,dt,new_dt,elapsed_time, input_file !Copy the RHS (contains state variables at current timestep) into chtkmp Call Get_All_RHS(chktmp%p1a) - ! Next, convert the AB terms into chebyshev space and copy them into the buffer - Call abterms_cheby%construct('p1a') - abterms_cheby%config='p1a' - !chktmp%p1a(:,:,:,numfields+1:numfields*2) = abterms(:,:,:,1:numfields) ! earlier version -- physical space storage - Call gridcp%to_spectral(abterms,abterms_cheby%p1a) - chktmp%p1a(:,:,:,numfields+1:numfields*2) = abterms_cheby%p1a(:,:,:,1:numfields) - Call abterms_cheby%deconstruct('p1a') + If ( (chebyshev) .and. (ndomains .eq. 1) ) Then + ! Convert the AB terms into chebyshev space and copy them into the buffer + ! For now, if ndomains > 1, we do not perform this conversion + Call abterms_cheby%construct('p1a') + abterms_cheby%config='p1a' + !chktmp%p1a(:,:,:,numfields+1:numfields*2) = abterms(:,:,:,1:numfields) ! earlier version -- physical space storage + Call gridcp%to_spectral(abterms,abterms_cheby%p1a) + chktmp%p1a(:,:,:,numfields+1:numfields*2) = abterms_cheby%p1a(:,:,:,1:numfields) + Call abterms_cheby%deconstruct('p1a') + Endif + !Move checkpoint buffer from p1a to s2a (rlm space) Call chktmp%reform() @@ -620,7 +624,8 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Endif - If (version .ge. 3) Then + + If ( (version .ge. 3) .and. chebyshev ) Then ! For version 3+, the AB terms are stored in Chebyshev space Call abterms_cheby%construct('p1b') abterms_cheby%config='p1b' @@ -633,15 +638,18 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) If (ndomains .eq. 1) Then ! Regardless of the checkpoint version or resolution, we can load the state vector ! fields directly from the checkpoint buffer because that portion of the buffer is - ! already in the expected Chebyshev format. + ! already in the expected format. ub = Min(n_r,n_r_old) fields = 0d0 fields(1:ub,:,:,1:numfields) = chktmp%p1b(1:ub,:,:,1:numfields) - ! For checkpoint version 3 and later, the AB terms are also stored in - ! Chebyshev space, but they need to be in grid space. - ! Even if the resolution changes, no further work needs to be done. - If (version .ge. 3) Then + If (.not. chebyshev) Then + abterms(:,:,:,1:numfields) = chktmp%p1b(:,:,:,numfields+1:numfields*2) + + Else If (version .ge. 3) Then + ! For checkpoint version 3 and later, the AB terms are also stored in + ! Chebyshev space, but they need to be in grid space. + ! Even if the resolution changes, no further work needs to be done. !Need to convert from Chebyshev space to physical space Call gridcp%From_Spectral(abterms_cheby%p1b,abterms) Call abterms_cheby%deconstruct('p1b') @@ -740,7 +748,7 @@ Subroutine Read_Checkpoint(fields, abterms,iteration,read_pars) Endif Else ! ndomains > 1: we need to loop over domains and (maybe) interpolate - + ! Note that ndomains > 1 is not possible when running in finite-difference mode. ! Loop over the domains to set the Chebyshev coefficients, ! possibly interpolating in radius for each subdomain ! The fields are easy, since they are stored in spectral (Chebyshev) space From ccb120f52f350c603847f429e44e679d367e53f3 Mon Sep 17 00:00:00 2001 From: Nick Featherstone Date: Tue, 11 Aug 2026 12:34:16 -0600 Subject: [PATCH 28/28] Changed init_type to 1 and ell values = 0,4 --- tests/checkpoint/old_format/main_input | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/checkpoint/old_format/main_input b/tests/checkpoint/old_format/main_input index 66f3d4c8d..3065b2ee6 100644 --- a/tests/checkpoint/old_format/main_input +++ b/tests/checkpoint/old_format/main_input @@ -31,7 +31,7 @@ ! New equatorial slices output -sph_mode_ell = 2,4,8, 32 +sph_mode_ell = 0,4 sph_mode_levels = 1 ! outer boundary sph_mode_values = 507 ! radial entropy gradient sph_mode_frequency = 10 @@ -56,7 +56,7 @@ fix_tvar_top = .true. fix_tvar_bottom = .true. / &Initial_Conditions_Namelist -init_type=7 ! Benchmark init +init_type=1 ! Benchmark init temp_amp = 1.0d-3 temp_w = 0.01d4 restart_iter=-1